diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv index 50ccc31bba986f1669c414570af5bc44c2f01e8f..de3443bf7c53b3ce84e8adf79b25cd0ffb02b20b 100644 --- a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv @@ -150,3 +150,75 @@ 5, 85043 6, 85053 7, 85041 +0, 86105 +1, 86165 +2, 86161 +3, 86175 +4, 86075 +5, 86199 +6, 86167 +7, 86219 +0, 87195 +1, 87305 +2, 87271 +3, 87285 +4, 87213 +5, 87283 +6, 87299 +7, 87317 +0, 110343 +1, 110333 +2, 110333 +3, 110333 +4, 110319 +5, 110333 +6, 110333 +7, 110333 +0, 110343 +1, 110333 +2, 110333 +3, 110333 +4, 110319 +5, 110333 +6, 110333 +7, 110333 +0, 110343 +1, 110333 +2, 110333 +3, 110333 +4, 110319 +5, 110333 +6, 110333 +7, 110333 +0, 105951 +1, 106203 +2, 106203 +3, 105991 +4, 105863 +5, 105557 +6, 105621 +7, 106203 +0, 104139 +1, 103553 +2, 104135 +3, 103873 +4, 104185 +5, 104781 +6, 104519 +7, 104781 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 103101 +6, 4 +7, 103101 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log index b32dfa1cfe3409fec77a80413ef4b2320e067a5e..4eab436d7c55f92553e59d7dce8fd59eca0119c7 100644 --- a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log @@ -1114,3 +1114,7836 @@ WARNING:megatron.core.rerun_state_machine:Result validation enabled warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65485 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65245 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65485 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65245 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65598 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65527 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65261 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65387 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65621 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65261 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=40960 expected_recv=65387 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65309 expected_overflow=0 exact_recv_per_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65574 expected_overflow=0 exact_recv_per_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65570 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65493 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65309 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65574 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65708 expected_overflow=0 exact_recv_per_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65475 expected_overflow=0 exact_recv_per_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65708 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65475 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65672 expected_overflow=0 exact_recv_per_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65685 expected_overflow=0 exact_recv_per_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65319 expected_overflow=0 exact_recv_per_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65504 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65511 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65672 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65685 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65319 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65308 expected_overflow=0 exact_recv_per_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65308 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65685 expected_overflow=0 exact_recv_per_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65393 expected_overflow=0 exact_recv_per_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65588 expected_overflow=0 exact_recv_per_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65685 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65639 expected_overflow=0 exact_recv_per_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65393 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65315 expected_overflow=0 exact_recv_per_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65639 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65588 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65516 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65315 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65269 expected_overflow=0 exact_recv_per_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65269 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65428 expected_overflow=0 exact_recv_per_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65519 expected_overflow=0 exact_recv_per_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65643 expected_overflow=0 exact_recv_per_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65485 expected_overflow=0 exact_recv_per_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65428 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65519 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65837 expected_overflow=0 exact_recv_per_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65718 expected_overflow=0 exact_recv_per_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65643 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65485 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65837 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65718 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65079 expected_overflow=0 exact_recv_per_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65079 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65839 expected_overflow=0 exact_recv_per_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65267 expected_overflow=0 exact_recv_per_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66164 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65681 expected_overflow=0 exact_recv_per_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65596 expected_overflow=0 exact_recv_per_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65404 expected_overflow=0 exact_recv_per_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65839 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65267 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65445 expected_overflow=0 exact_recv_per_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65681 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65636 expected_overflow=0 exact_recv_per_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65596 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65404 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65445 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66164 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65636 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65314 expected_overflow=0 exact_recv_per_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65243 expected_overflow=0 exact_recv_per_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65484 expected_overflow=0 exact_recv_per_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65314 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65659 expected_overflow=0 exact_recv_per_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65541 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65481 expected_overflow=0 exact_recv_per_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65243 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65484 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65659 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65481 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65597 expected_overflow=0 exact_recv_per_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65487 expected_overflow=0 exact_recv_per_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65597 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65487 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65434 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65881 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65865 expected_overflow=0 exact_recv_per_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65317 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65054 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65434 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65881 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65865 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65317 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65054 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65899 expected_overflow=0 exact_recv_per_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65489 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65899 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65489 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65961 expected_overflow=0 exact_recv_per_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65518 expected_overflow=0 exact_recv_per_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65961 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65577 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65518 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65456 expected_overflow=0 exact_recv_per_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65904 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65723 expected_overflow=0 exact_recv_per_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65456 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65904 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65723 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65428 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65779 expected_overflow=0 exact_recv_per_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65428 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65779 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65737 expected_overflow=0 exact_recv_per_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65282 expected_overflow=0 exact_recv_per_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65780 expected_overflow=0 exact_recv_per_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65737 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65282 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65780 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65728 expected_overflow=0 exact_recv_per_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65126 expected_overflow=0 exact_recv_per_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65925 expected_overflow=0 exact_recv_per_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65728 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65126 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65315 expected_overflow=0 exact_recv_per_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65615 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65925 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65315 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65482 expected_overflow=0 exact_recv_per_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65309 expected_overflow=0 exact_recv_per_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65614 expected_overflow=0 exact_recv_per_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65482 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65309 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65614 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65647 expected_overflow=0 exact_recv_per_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65273 expected_overflow=0 exact_recv_per_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65432 expected_overflow=0 exact_recv_per_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65647 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65596 expected_overflow=0 exact_recv_per_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65273 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65432 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65596 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65501 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65641 expected_overflow=0 exact_recv_per_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65213 expected_overflow=0 exact_recv_per_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65698 expected_overflow=0 exact_recv_per_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=40960 expected_recv=65641 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65213 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65698 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65502 expected_overflow=0 exact_recv_per_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65762 expected_overflow=0 exact_recv_per_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65565 expected_overflow=0 exact_recv_per_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65502 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65398 expected_overflow=0 exact_recv_per_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65762 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65598 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65565 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65398 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65227 expected_overflow=0 exact_recv_per_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65335 expected_overflow=0 exact_recv_per_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65227 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65455 expected_overflow=0 exact_recv_per_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65335 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65468 expected_overflow=0 exact_recv_per_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65455 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65898 expected_overflow=0 exact_recv_per_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65669 expected_overflow=0 exact_recv_per_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65721 expected_overflow=0 exact_recv_per_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65468 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65253 expected_overflow=0 exact_recv_per_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65898 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65669 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65721 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65253 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65781 expected_overflow=0 exact_recv_per_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65781 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65407 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65930 expected_overflow=0 exact_recv_per_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65507 expected_overflow=0 exact_recv_per_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65930 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65420 expected_overflow=0 exact_recv_per_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65620 expected_overflow=0 exact_recv_per_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65507 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65188 expected_overflow=0 exact_recv_per_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65420 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65530 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65620 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65188 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65184 expected_overflow=0 exact_recv_per_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65925 expected_overflow=0 exact_recv_per_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65184 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65757 expected_overflow=0 exact_recv_per_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65925 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65127 expected_overflow=0 exact_recv_per_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65757 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65450 expected_overflow=0 exact_recv_per_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65603 expected_overflow=0 exact_recv_per_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65382 expected_overflow=0 exact_recv_per_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65127 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65153 expected_overflow=0 exact_recv_per_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65450 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65603 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65382 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65153 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65699 expected_overflow=0 exact_recv_per_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65456 expected_overflow=0 exact_recv_per_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=40960 expected_recv=65699 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65226 expected_overflow=0 exact_recv_per_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65456 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65226 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65477 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65436 expected_overflow=0 exact_recv_per_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65142 expected_overflow=0 exact_recv_per_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65615 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65598 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65436 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65142 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65625 expected_overflow=0 exact_recv_per_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65625 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33274, 32641, 32746, 32688, 32846, 32732, 32485, 32732] avg=32768.0000 max=33274 max_rank=0 min=32485 min_rank=6 imbalance_ratio=1.015442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33274, 32641, 32746, 32688, 32846, 32732, 32485, 32732] avg=32768.0000 max=33274 max_rank=0 min=32485 min_rank=6 imbalance_ratio=1.015442 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65948 expected_overflow=0 exact_recv_per_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[33274, 32641, 32746, 32688, 32846, 32732, 32485, 32732] avg=32768.0000 max=33274 max_rank=0 min=32485 min_rank=6 imbalance_ratio=1.015442 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65631 expected_overflow=0 exact_recv_per_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65948 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33274, 32641, 32746, 32688, 32846, 32732, 32485, 32732] avg=32768.0000 max=33274 max_rank=0 min=32485 min_rank=6 imbalance_ratio=1.015442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] avg=32768.0000 max=33096 max_rank=0 min=32641 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=178 max_load_reduction_pct=0.005350 moved_to_replicas=178 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33274, 32641, 32746, 32688, 32846, 32732, 32485, 32732] avg=32768.0000 max=33274 max_rank=0 min=32485 min_rank=6 imbalance_ratio=1.015442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] avg=32768.0000 max=33096 max_rank=0 min=32641 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=178 max_load_reduction_pct=0.005350 moved_to_replicas=178 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65790 expected_overflow=0 exact_recv_per_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65631 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] avg=32768.0000 max=33096 max_rank=0 min=32641 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=178 max_load_reduction_pct=0.005350 moved_to_replicas=178 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65790 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] avg=32768.0000 max=33096 max_rank=0 min=32641 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=178 max_load_reduction_pct=0.005350 moved_to_replicas=178 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] avg=32768.0000 max=33096 max_rank=0 min=32641 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=178 max_load_reduction_pct=0.005350 moved_to_replicas=178 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65603 expected_overflow=0 exact_recv_per_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33274, 32641, 32746, 32688, 32846, 32732, 32485, 32732] avg=32768.0000 max=33274 max_rank=0 min=32485 min_rank=6 imbalance_ratio=1.015442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33274, 32641, 32746, 32688, 32846, 32732, 32485, 32732] avg=32768.0000 max=33274 max_rank=0 min=32485 min_rank=6 imbalance_ratio=1.015442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65603 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] avg=32768.0000 max=33096 max_rank=0 min=32641 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=178 max_load_reduction_pct=0.005350 moved_to_replicas=178 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] avg=32768.0000 max=33096 max_rank=0 min=32641 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=178 max_load_reduction_pct=0.005350 moved_to_replicas=178 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65959 expected_overflow=0 exact_recv_per_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=40960 expected_recv=65959 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32573, 33115, 32682, 32763] avg=32768.0000 max=33115 max_rank=5 min=32573 min_rank=4 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65848 expected_overflow=0 exact_recv_per_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65848 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32573, 33115, 32682, 32763] avg=32768.0000 max=33115 max_rank=5 min=32573 min_rank=4 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32573, 33115, 32682, 32763] avg=32768.0000 max=33115 max_rank=5 min=32573 min_rank=4 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32573, 33115, 32682, 32763] avg=32768.0000 max=33115 max_rank=5 min=32573 min_rank=4 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] avg=32768.0000 max=33096 max_rank=5 min=32592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32573, 33115, 32682, 32763] avg=32768.0000 max=33115 max_rank=5 min=32573 min_rank=4 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] avg=32768.0000 max=33096 max_rank=5 min=32592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] avg=32768.0000 max=33096 max_rank=5 min=32592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] avg=32768.0000 max=33096 max_rank=5 min=32592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] avg=32768.0000 max=33096 max_rank=5 min=32592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65337 expected_overflow=0 exact_recv_per_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65337 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32573, 33115, 32682, 32763] avg=32768.0000 max=33115 max_rank=5 min=32573 min_rank=4 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32573, 33115, 32682, 32763] avg=32768.0000 max=33115 max_rank=5 min=32573 min_rank=4 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] avg=32768.0000 max=33096 max_rank=5 min=32592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33274, 32641, 32746, 32688, 32846, 32732, 32485, 32732] avg=32768.0000 max=33274 max_rank=0 min=32485 min_rank=6 imbalance_ratio=1.015442 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] avg=32768.0000 max=33096 max_rank=5 min=32592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33096, 32641, 32746, 32688, 32846, 32732, 32663, 32732] avg=32768.0000 max=33096 max_rank=0 min=32641 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=178 max_load_reduction_pct=0.005350 moved_to_replicas=178 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32678, 32875, 32925, 32828, 32775, 32849, 32560, 32654] avg=32768.0000 max=32925 max_rank=2 min=32560 min_rank=6 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[33137, 32617, 32773, 32808, 32687, 32640, 32463, 33019] avg=32768.0000 max=33137 max_rank=0 min=32463 min_rank=6 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[33137, 32617, 32773, 32808, 32687, 32640, 32463, 33019] avg=32768.0000 max=33137 max_rank=0 min=32463 min_rank=6 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[33137, 32617, 32773, 32808, 32687, 32640, 32463, 33019] avg=32768.0000 max=33137 max_rank=0 min=32463 min_rank=6 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] avg=32768.0000 max=33096 max_rank=0 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[33137, 32617, 32773, 32808, 32687, 32640, 32463, 33019] avg=32768.0000 max=33137 max_rank=0 min=32463 min_rank=6 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[33137, 32617, 32773, 32808, 32687, 32640, 32463, 33019] avg=32768.0000 max=33137 max_rank=0 min=32463 min_rank=6 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] avg=32768.0000 max=33096 max_rank=0 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[32983, 32579, 32689, 32827, 32974, 32532, 32812, 32748] avg=32768.0000 max=32983 max_rank=0 min=32532 min_rank=5 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] avg=32768.0000 max=33096 max_rank=0 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] avg=32768.0000 max=33096 max_rank=0 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] avg=32768.0000 max=33096 max_rank=0 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32573, 33115, 32682, 32763] avg=32768.0000 max=33115 max_rank=5 min=32573 min_rank=4 imbalance_ratio=1.010590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[33137, 32617, 32773, 32808, 32687, 32640, 32463, 33019] avg=32768.0000 max=33137 max_rank=0 min=32463 min_rank=6 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[32892, 32776, 32648, 32695, 32592, 33096, 32682, 32763] avg=32768.0000 max=33096 max_rank=5 min=32592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19 max_load_reduction_pct=0.000574 moved_to_replicas=19 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] avg=32768.0000 max=33096 max_rank=0 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[33137, 32617, 32773, 32808, 32687, 32640, 32463, 33019] avg=32768.0000 max=33137 max_rank=0 min=32463 min_rank=6 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] avg=32768.0000 max=33096 max_rank=0 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[32731, 32766, 32549, 33045, 32708, 32779, 32832, 32734] avg=32768.0000 max=33045 max_rank=3 min=32549 min_rank=2 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32880, 32787, 32918, 32725, 32707, 32629, 32659, 32839] avg=32768.0000 max=32918 max_rank=2 min=32629 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32714, 32930, 32628, 32977, 32709, 32868, 32733, 32585] avg=32768.0000 max=32977 max_rank=3 min=32585 min_rank=7 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[33137, 32617, 32773, 32808, 32687, 32640, 32463, 33019] avg=32768.0000 max=33137 max_rank=0 min=32463 min_rank=6 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[33096, 32617, 32773, 32808, 32687, 32640, 32504, 33019] avg=32768.0000 max=33096 max_rank=0 min=32504 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32724, 32687, 32683, 32610, 32789, 33007, 32808, 32836] avg=32768.0000 max=33007 max_rank=5 min=32610 min_rank=3 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[32781, 32877, 32543, 32845, 32791, 32727, 32680, 32900] avg=32768.0000 max=32900 max_rank=7 min=32543 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[32863, 32908, 32651, 32898, 32390, 32626, 32834, 32974] avg=32768.0000 max=32974 max_rank=7 min=32390 min_rank=4 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[32844, 32928, 32993, 32901, 32401, 32371, 32777, 32929] avg=32768.0000 max=32993 max_rank=2 min=32371 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[32965, 32716, 32801, 32651, 32796, 32433, 33040, 32742] avg=32768.0000 max=33040 max_rank=6 min=32433 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=1047984 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254848 peak_demand=32957 required_capacity=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=1046990 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255744 peak_demand=32995 required_capacity=1.006927 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[32980, 32900, 32719, 32473, 32826, 32413, 32856, 32977] avg=32768.0000 max=32980 max_rank=0 min=32413 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=1048637 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253824 peak_demand=33040 required_capacity=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=1048183 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253696 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=1049485 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253312 peak_demand=33019 required_capacity=1.007660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[32788, 32646, 32607, 32794, 32995, 32657, 32660, 32997] avg=32768.0000 max=32997 max_rank=7 min=32607 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[32724, 32717, 32815, 32836, 32824, 32919, 32726, 32583] avg=32768.0000 max=32919 max_rank=5 min=32583 min_rank=7 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=1047907 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254592 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=1048897 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253824 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1050525 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251520 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:12:53] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 26436.2 | throughput per GPU (TFLOP/s/GPU): 12.6 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102423E+01 | loss scale: 1.0 | grad norm: 43.574 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58091.849609375 | max allocated: 76371.875 | reserved: 78094.0 | max reserved: 78094.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65613 expected_overflow=0 exact_recv_per_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65373 expected_overflow=0 exact_recv_per_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65804 expected_overflow=0 exact_recv_per_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65662 expected_overflow=0 exact_recv_per_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65394 expected_overflow=0 exact_recv_per_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65673 expected_overflow=0 exact_recv_per_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65299 expected_overflow=0 exact_recv_per_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65613 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65373 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65804 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65662 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65394 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65673 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65299 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65470 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65921 expected_overflow=0 exact_recv_per_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65921 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65614 expected_overflow=0 exact_recv_per_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65345 expected_overflow=0 exact_recv_per_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65608 expected_overflow=0 exact_recv_per_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65614 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65554 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65401 expected_overflow=0 exact_recv_per_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65299 expected_overflow=0 exact_recv_per_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65345 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65608 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65618 expected_overflow=0 exact_recv_per_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65401 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65299 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65618 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65682 expected_overflow=0 exact_recv_per_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65910 expected_overflow=0 exact_recv_per_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65740 expected_overflow=0 exact_recv_per_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65194 expected_overflow=0 exact_recv_per_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65682 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65156 expected_overflow=0 exact_recv_per_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65910 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65740 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65743 expected_overflow=0 exact_recv_per_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65245 expected_overflow=0 exact_recv_per_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65627 expected_overflow=0 exact_recv_per_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65194 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65156 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65743 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65245 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65627 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65650 expected_overflow=0 exact_recv_per_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65965 expected_overflow=0 exact_recv_per_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65650 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65076 expected_overflow=0 exact_recv_per_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65512 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65585 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65345 expected_overflow=0 exact_recv_per_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65965 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65528 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65076 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65345 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65580 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65746 expected_overflow=0 exact_recv_per_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65440 expected_overflow=0 exact_recv_per_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65176 expected_overflow=0 exact_recv_per_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65746 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65440 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65176 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65667 expected_overflow=0 exact_recv_per_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65379 expected_overflow=0 exact_recv_per_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65817 expected_overflow=0 exact_recv_per_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65667 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65483 expected_overflow=0 exact_recv_per_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65379 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65783 expected_overflow=0 exact_recv_per_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65817 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=40960 expected_recv=65483 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65783 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65464 expected_overflow=0 exact_recv_per_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65350 expected_overflow=0 exact_recv_per_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65253 expected_overflow=0 exact_recv_per_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65464 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65350 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65253 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65586 expected_overflow=0 exact_recv_per_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65675 expected_overflow=0 exact_recv_per_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65441 expected_overflow=0 exact_recv_per_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65586 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65636 expected_overflow=0 exact_recv_per_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65736 expected_overflow=0 exact_recv_per_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65675 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65441 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65636 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65736 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65188 expected_overflow=0 exact_recv_per_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65822 expected_overflow=0 exact_recv_per_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65188 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65327 expected_overflow=0 exact_recv_per_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65822 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65656 expected_overflow=0 exact_recv_per_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65327 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65656 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65718 expected_overflow=0 exact_recv_per_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65302 expected_overflow=0 exact_recv_per_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65300 expected_overflow=0 exact_recv_per_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65639 expected_overflow=0 exact_recv_per_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65718 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65300 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65302 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65639 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65514 expected_overflow=0 exact_recv_per_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65304 expected_overflow=0 exact_recv_per_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65686 expected_overflow=0 exact_recv_per_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65556 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65514 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65304 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65686 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65339 expected_overflow=0 exact_recv_per_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65567 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65666 expected_overflow=0 exact_recv_per_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65695 expected_overflow=0 exact_recv_per_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65339 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65666 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65695 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65302 expected_overflow=0 exact_recv_per_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65748 expected_overflow=0 exact_recv_per_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65302 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65506 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65748 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65747 expected_overflow=0 exact_recv_per_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65574 expected_overflow=0 exact_recv_per_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65811 expected_overflow=0 exact_recv_per_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65747 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65293 expected_overflow=0 exact_recv_per_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65574 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65811 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=40960 expected_recv=65293 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65583 expected_overflow=0 exact_recv_per_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65562 expected_overflow=0 exact_recv_per_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65787 expected_overflow=0 exact_recv_per_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65583 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65547 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65787 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65562 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65238 expected_overflow=0 exact_recv_per_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65238 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65671 expected_overflow=0 exact_recv_per_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65326 expected_overflow=0 exact_recv_per_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65407 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65671 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65326 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65737 expected_overflow=0 exact_recv_per_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65466 expected_overflow=0 exact_recv_per_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65334 expected_overflow=0 exact_recv_per_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65607 expected_overflow=0 exact_recv_per_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65737 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65466 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65334 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65607 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65333 expected_overflow=0 exact_recv_per_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65571 expected_overflow=0 exact_recv_per_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65333 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65691 expected_overflow=0 exact_recv_per_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65713 expected_overflow=0 exact_recv_per_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65571 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65691 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=40960 expected_recv=65713 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65418 expected_overflow=0 exact_recv_per_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65384 expected_overflow=0 exact_recv_per_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65443 expected_overflow=0 exact_recv_per_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65793 expected_overflow=0 exact_recv_per_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65418 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65384 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65443 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65793 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65334 expected_overflow=0 exact_recv_per_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65338 expected_overflow=0 exact_recv_per_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65334 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65338 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65608 expected_overflow=0 exact_recv_per_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65737 expected_overflow=0 exact_recv_per_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65608 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=40960 expected_recv=65737 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65775 expected_overflow=0 exact_recv_per_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65347 expected_overflow=0 exact_recv_per_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65432 expected_overflow=0 exact_recv_per_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65521 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65775 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65347 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65434 expected_overflow=0 exact_recv_per_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65432 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65434 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65435 expected_overflow=0 exact_recv_per_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65929 expected_overflow=0 exact_recv_per_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65435 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65929 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65512 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65436 expected_overflow=0 exact_recv_per_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65156 expected_overflow=0 exact_recv_per_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65601 expected_overflow=0 exact_recv_per_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65436 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65156 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65601 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65854 expected_overflow=0 exact_recv_per_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65688 expected_overflow=0 exact_recv_per_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65854 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65231 expected_overflow=0 exact_recv_per_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65688 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65616 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65887 expected_overflow=0 exact_recv_per_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65231 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65887 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65361 expected_overflow=0 exact_recv_per_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65362 expected_overflow=0 exact_recv_per_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65549 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65361 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65362 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65574 expected_overflow=0 exact_recv_per_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65638 expected_overflow=0 exact_recv_per_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65599 expected_overflow=0 exact_recv_per_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65574 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65638 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65401 expected_overflow=0 exact_recv_per_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65599 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65826 expected_overflow=0 exact_recv_per_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65401 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65364 expected_overflow=0 exact_recv_per_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65653 expected_overflow=0 exact_recv_per_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65826 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65364 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65553 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65707 expected_overflow=0 exact_recv_per_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65653 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65707 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65344 expected_overflow=0 exact_recv_per_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65344 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65496 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32614, 32732, 32909, 32859, 32876, 32392, 32876, 32886] avg=32768.0000 max=32909 max_rank=2 min=32392 min_rank=5 imbalance_ratio=1.004303 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[32663, 33053, 32630, 32929, 32514, 32752, 32764, 32839] avg=32768.0000 max=33053 max_rank=1 min=32514 min_rank=4 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[32811, 32782, 32647, 32896, 32893, 32745, 32867, 32503] avg=32768.0000 max=32896 max_rank=3 min=32503 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33234, 32791, 32854, 32317] avg=32768.0000 max=33234 max_rank=4 min=32317 min_rank=7 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33234, 32791, 32854, 32317] avg=32768.0000 max=33234 max_rank=4 min=32317 min_rank=7 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] avg=32768.0000 max=33096 max_rank=4 min=32455 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33234, 32791, 32854, 32317] avg=32768.0000 max=33234 max_rank=4 min=32317 min_rank=7 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33234, 32791, 32854, 32317] avg=32768.0000 max=33234 max_rank=4 min=32317 min_rank=7 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] avg=32768.0000 max=33096 max_rank=4 min=32455 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33234, 32791, 32854, 32317] avg=32768.0000 max=33234 max_rank=4 min=32317 min_rank=7 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[32748, 32663, 32837, 32838, 32704, 32513, 32953, 32888] avg=32768.0000 max=32953 max_rank=6 min=32513 min_rank=5 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] avg=32768.0000 max=33096 max_rank=4 min=32455 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] avg=32768.0000 max=33096 max_rank=4 min=32455 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] avg=32768.0000 max=33096 max_rank=4 min=32455 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33234, 32791, 32854, 32317] avg=32768.0000 max=33234 max_rank=4 min=32317 min_rank=7 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33234, 32791, 32854, 32317] avg=32768.0000 max=33234 max_rank=4 min=32317 min_rank=7 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33234, 32791, 32854, 32317] avg=32768.0000 max=33234 max_rank=4 min=32317 min_rank=7 imbalance_ratio=1.014221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] avg=32768.0000 max=33096 max_rank=4 min=32455 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] avg=32768.0000 max=33096 max_rank=4 min=32455 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[32862, 32640, 32776, 32670, 33096, 32791, 32854, 32455] avg=32768.0000 max=33096 max_rank=4 min=32455 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=138 max_load_reduction_pct=0.004152 moved_to_replicas=138 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32615, 32965, 32765, 32745, 32907, 32815, 32967, 32365] avg=32768.0000 max=32967 max_rank=6 min=32365 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[32676, 32865, 32837, 32729, 32524, 32796, 32800, 32917] avg=32768.0000 max=32917 max_rank=7 min=32524 min_rank=4 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[32563, 32658, 32799, 32724, 32879, 32982, 32816, 32723] avg=32768.0000 max=32982 max_rank=5 min=32563 min_rank=0 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[32670, 32660, 32798, 32867, 32900, 32547, 32808, 32894] avg=32768.0000 max=32900 max_rank=4 min=32547 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[32771, 32680, 32825, 32907, 32933, 32577, 32731, 32720] avg=32768.0000 max=32933 max_rank=4 min=32577 min_rank=5 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[32751, 32538, 32601, 32773, 32709, 33072, 32694, 33006] avg=32768.0000 max=33072 max_rank=5 min=32538 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[32837, 32746, 32711, 32881, 32644, 32682, 32701, 32942] avg=32768.0000 max=32942 max_rank=7 min=32644 min_rank=4 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[32651, 32750, 32660, 32768, 33080, 32846, 32905, 32484] avg=32768.0000 max=33080 max_rank=4 min=32484 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[32595, 32599, 32921, 33036, 32597, 33038, 32572, 32786] avg=32768.0000 max=33038 max_rank=5 min=32572 min_rank=6 imbalance_ratio=1.008240 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[32734, 32727, 32662, 32748, 32661, 32877, 32819, 32916] avg=32768.0000 max=32916 max_rank=7 min=32661 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[32626, 32928, 32837, 32849, 32571, 32819, 32791, 32723] avg=32768.0000 max=32928 max_rank=1 min=32571 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=1049118 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253824 peak_demand=33056 required_capacity=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=1048996 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253440 peak_demand=33053 required_capacity=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=1048514 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254336 peak_demand=32978 required_capacity=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=1047791 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255360 peak_demand=33006 required_capacity=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=1048042 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254848 peak_demand=32951 required_capacity=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=1048500 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254208 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=1049014 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254208 peak_demand=33036 required_capacity=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=1048633 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253568 peak_demand=33072 required_capacity=1.009277 + [2026-08-30 05:12:53] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 594.6 | throughput per GPU (TFLOP/s/GPU): 560.7 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101991E+01 | loss scale: 1.0 | grad norm: 45.257 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65228 expected_overflow=0 exact_recv_per_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65415 expected_overflow=0 exact_recv_per_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65627 expected_overflow=0 exact_recv_per_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65790 expected_overflow=0 exact_recv_per_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65228 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65706 expected_overflow=0 exact_recv_per_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65627 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65404 expected_overflow=0 exact_recv_per_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65415 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65580 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65538 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65790 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65706 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65404 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65383 expected_overflow=0 exact_recv_per_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65174 expected_overflow=0 exact_recv_per_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65383 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65670 expected_overflow=0 exact_recv_per_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65397 expected_overflow=0 exact_recv_per_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65722 expected_overflow=0 exact_recv_per_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65174 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65808 expected_overflow=0 exact_recv_per_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65670 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65397 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65722 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65604 expected_overflow=0 exact_recv_per_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65808 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65530 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65604 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65781 expected_overflow=0 exact_recv_per_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65781 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65331 expected_overflow=0 exact_recv_per_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65336 expected_overflow=0 exact_recv_per_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65331 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65448 expected_overflow=0 exact_recv_per_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65512 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65336 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65448 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65545 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65777 expected_overflow=0 exact_recv_per_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65777 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65621 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65653 expected_overflow=0 exact_recv_per_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65465 expected_overflow=0 exact_recv_per_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65446 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65653 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65261 expected_overflow=0 exact_recv_per_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65465 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65809 expected_overflow=0 exact_recv_per_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65703 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65261 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65330 expected_overflow=0 exact_recv_per_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65809 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65752 expected_overflow=0 exact_recv_per_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65330 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65752 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65630 expected_overflow=0 exact_recv_per_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65630 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65301 expected_overflow=0 exact_recv_per_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65501 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65301 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65213 expected_overflow=0 exact_recv_per_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65545 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65826 expected_overflow=0 exact_recv_per_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65213 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65520 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65284 expected_overflow=0 exact_recv_per_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65826 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65284 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65735 expected_overflow=0 exact_recv_per_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65735 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65469 expected_overflow=0 exact_recv_per_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65469 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65482 expected_overflow=0 exact_recv_per_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65616 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65639 expected_overflow=0 exact_recv_per_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65396 expected_overflow=0 exact_recv_per_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65482 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65639 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65667 expected_overflow=0 exact_recv_per_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65396 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65475 expected_overflow=0 exact_recv_per_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=40960 expected_recv=65667 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65475 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65314 expected_overflow=0 exact_recv_per_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65314 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=40960 expected_recv=66066 expected_overflow=0 exact_recv_per_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65615 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=40960 expected_recv=66066 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65172 expected_overflow=0 exact_recv_per_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65544 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65254 expected_overflow=0 exact_recv_per_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65172 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65544 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65254 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65415 expected_overflow=0 exact_recv_per_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65336 expected_overflow=0 exact_recv_per_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65415 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65478 expected_overflow=0 exact_recv_per_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65336 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65478 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66104 expected_overflow=0 exact_recv_per_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65600 expected_overflow=0 exact_recv_per_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66104 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65355 expected_overflow=0 exact_recv_per_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65590 expected_overflow=0 exact_recv_per_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65600 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65511 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65355 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65590 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65718 expected_overflow=0 exact_recv_per_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65639 expected_overflow=0 exact_recv_per_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65718 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65589 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65639 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65307 expected_overflow=0 exact_recv_per_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65896 expected_overflow=0 exact_recv_per_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65269 expected_overflow=0 exact_recv_per_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65307 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65465 expected_overflow=0 exact_recv_per_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65896 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65269 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65465 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=40960 expected_recv=65515 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65562 expected_overflow=0 exact_recv_per_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65384 expected_overflow=0 exact_recv_per_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65161 expected_overflow=0 exact_recv_per_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65562 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65384 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65161 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65929 expected_overflow=0 exact_recv_per_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65482 expected_overflow=0 exact_recv_per_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65929 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65828 expected_overflow=0 exact_recv_per_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65477 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65482 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65828 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65037 expected_overflow=0 exact_recv_per_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65473 expected_overflow=0 exact_recv_per_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66010 expected_overflow=0 exact_recv_per_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65037 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65473 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66010 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65515 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65782 expected_overflow=0 exact_recv_per_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65855 expected_overflow=0 exact_recv_per_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65468 expected_overflow=0 exact_recv_per_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65782 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65855 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65470 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65468 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65326 expected_overflow=0 exact_recv_per_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=40960 expected_recv=66130 expected_overflow=0 exact_recv_per_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65784 expected_overflow=0 exact_recv_per_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65326 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=40960 expected_recv=66130 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65784 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65450 expected_overflow=0 exact_recv_per_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65450 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65016 expected_overflow=0 exact_recv_per_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65701 expected_overflow=0 exact_recv_per_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65016 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65701 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65309 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65026 expected_overflow=0 exact_recv_per_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65892 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65576 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65309 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65026 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65892 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65576 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65358 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65358 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65321 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65602 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65740 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65321 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65740 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65943 expected_overflow=0 exact_recv_per_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65842 expected_overflow=0 exact_recv_per_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65324 expected_overflow=0 exact_recv_per_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65560 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65943 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65842 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65324 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65560 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65259 expected_overflow=0 exact_recv_per_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65688 expected_overflow=0 exact_recv_per_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65259 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65688 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65272 expected_overflow=0 exact_recv_per_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65548 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65272 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65659 expected_overflow=0 exact_recv_per_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65420 expected_overflow=0 exact_recv_per_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65659 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65420 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65564 expected_overflow=0 exact_recv_per_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=40960 expected_recv=65498 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65302 expected_overflow=0 exact_recv_per_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65564 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65302 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65506 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65195 expected_overflow=0 exact_recv_per_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65195 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65706 expected_overflow=0 exact_recv_per_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65619 expected_overflow=0 exact_recv_per_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65568 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65706 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65619 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65673 expected_overflow=0 exact_recv_per_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65292 expected_overflow=0 exact_recv_per_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65673 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65748 expected_overflow=0 exact_recv_per_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65292 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65748 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65551 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32788, 32787, 32705, 32595, 32726, 32846, 32929, 32768] avg=32768.0000 max=32929 max_rank=6 min=32595 min_rank=3 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32720, 32554, 32922, 33061, 32703, 32721, 32819, 32644] avg=32768.0000 max=33061 max_rank=3 min=32554 min_rank=1 imbalance_ratio=1.008942 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32722, 32696, 32783, 32693, 32695, 32902, 32672, 32981] avg=32768.0000 max=32981 max_rank=7 min=32672 min_rank=6 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[32848, 32713, 32935, 32726, 32745, 32688, 32726, 32763] avg=32768.0000 max=32935 max_rank=2 min=32688 min_rank=5 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[32544, 33084, 32869, 32821, 32648, 32821, 32807, 32550] avg=32768.0000 max=33084 max_rank=1 min=32544 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32655, 32738, 32751, 32712, 32887, 33059, 32771, 32571] avg=32768.0000 max=33059 max_rank=5 min=32571 min_rank=7 imbalance_ratio=1.008881 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32693, 32690, 32750, 32538, 32793, 32839, 33003, 32838] avg=32768.0000 max=33003 max_rank=6 min=32538 min_rank=3 imbalance_ratio=1.007172 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[32642, 32712, 32654, 32899, 33049, 32698, 32785, 32705] avg=32768.0000 max=33049 max_rank=4 min=32642 min_rank=0 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[32930, 32845, 32695, 32827, 32707, 32836, 32651, 32653] avg=32768.0000 max=32930 max_rank=0 min=32651 min_rank=6 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32488, 32715, 32694, 32808, 33124] avg=32768.0000 max=33124 max_rank=7 min=32488 min_rank=3 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32488, 32715, 32694, 32808, 33124] avg=32768.0000 max=33124 max_rank=7 min=32488 min_rank=3 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] avg=32768.0000 max=33096 max_rank=7 min=32516 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32488, 32715, 32694, 32808, 33124] avg=32768.0000 max=33124 max_rank=7 min=32488 min_rank=3 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32488, 32715, 32694, 32808, 33124] avg=32768.0000 max=33124 max_rank=7 min=32488 min_rank=3 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32488, 32715, 32694, 32808, 33124] avg=32768.0000 max=33124 max_rank=7 min=32488 min_rank=3 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] avg=32768.0000 max=33096 max_rank=7 min=32516 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[32897, 32759, 32897, 32742, 32852, 32758, 32559, 32680] avg=32768.0000 max=32897 max_rank=0 min=32559 min_rank=6 imbalance_ratio=1.003937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] avg=32768.0000 max=33096 max_rank=7 min=32516 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] avg=32768.0000 max=33096 max_rank=7 min=32516 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] avg=32768.0000 max=33096 max_rank=7 min=32516 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32488, 32715, 32694, 32808, 33124] avg=32768.0000 max=33124 max_rank=7 min=32488 min_rank=3 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32488, 32715, 32694, 32808, 33124] avg=32768.0000 max=33124 max_rank=7 min=32488 min_rank=3 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32488, 32715, 32694, 32808, 33124] avg=32768.0000 max=33124 max_rank=7 min=32488 min_rank=3 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] avg=32768.0000 max=33096 max_rank=7 min=32516 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] avg=32768.0000 max=33096 max_rank=7 min=32516 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[32956, 32547, 32812, 32516, 32715, 32694, 32808, 33096] avg=32768.0000 max=33096 max_rank=7 min=32516 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[32397, 33130, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33130 max_rank=1 min=32397 min_rank=0 imbalance_ratio=1.011047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[32397, 33130, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33130 max_rank=1 min=32397 min_rank=0 imbalance_ratio=1.011047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33096 max_rank=1 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34 max_load_reduction_pct=0.001026 moved_to_replicas=34 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[32397, 33130, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33130 max_rank=1 min=32397 min_rank=0 imbalance_ratio=1.011047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[32397, 33130, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33130 max_rank=1 min=32397 min_rank=0 imbalance_ratio=1.011047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33096 max_rank=1 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34 max_load_reduction_pct=0.001026 moved_to_replicas=34 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[32397, 33130, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33130 max_rank=1 min=32397 min_rank=0 imbalance_ratio=1.011047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33096 max_rank=1 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34 max_load_reduction_pct=0.001026 moved_to_replicas=34 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33096 max_rank=1 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34 max_load_reduction_pct=0.001026 moved_to_replicas=34 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[32428, 33088, 32803, 32556, 32602, 32875, 33087, 32705] avg=32768.0000 max=33088 max_rank=1 min=32428 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33096 max_rank=1 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34 max_load_reduction_pct=0.001026 moved_to_replicas=34 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[32397, 33130, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33130 max_rank=1 min=32397 min_rank=0 imbalance_ratio=1.011047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[32397, 33130, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33130 max_rank=1 min=32397 min_rank=0 imbalance_ratio=1.011047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[32397, 33130, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33130 max_rank=1 min=32397 min_rank=0 imbalance_ratio=1.011047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33096 max_rank=1 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34 max_load_reduction_pct=0.001026 moved_to_replicas=34 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33096 max_rank=1 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34 max_load_reduction_pct=0.001026 moved_to_replicas=34 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[32431, 33096, 32660, 32839, 32700, 33002, 32856, 32560] avg=32768.0000 max=33096 max_rank=1 min=32431 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34 max_load_reduction_pct=0.001026 moved_to_replicas=34 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[32880, 32848, 32893, 32778, 32546, 32480, 32686, 33033] avg=32768.0000 max=33033 max_rank=7 min=32480 min_rank=5 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[32896, 32750, 32840, 32860, 32679, 32624, 32687, 32808] avg=32768.0000 max=32896 max_rank=0 min=32624 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[32542, 32779, 32838, 32826, 32846, 32692, 32790, 32831] avg=32768.0000 max=32846 max_rank=4 min=32542 min_rank=0 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=1048110 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255104 peak_demand=33052 required_capacity=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=1049080 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253824 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=1048361 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253184 peak_demand=33087 required_capacity=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=1048911 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254208 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=1048333 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254592 peak_demand=33077 required_capacity=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=1048441 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254080 peak_demand=33002 required_capacity=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=1049511 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253312 peak_demand=33070 required_capacity=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=1047861 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254208 peak_demand=33083 required_capacity=1.009613 + [2026-08-30 05:12:54] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 426.2 | throughput per GPU (TFLOP/s/GPU): 782.2 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.034851E+01 | loss scale: 1.0 | grad norm: 209.982 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65154 expected_overflow=0 exact_recv_per_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65111 expected_overflow=0 exact_recv_per_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65752 expected_overflow=0 exact_recv_per_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65959 expected_overflow=0 exact_recv_per_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65427 expected_overflow=0 exact_recv_per_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65759 expected_overflow=0 exact_recv_per_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65568 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65154 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65111 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65752 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65959 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65427 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=40960 expected_recv=65759 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65707 expected_overflow=0 exact_recv_per_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65731 expected_overflow=0 exact_recv_per_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65983 expected_overflow=0 exact_recv_per_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65506 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65207 expected_overflow=0 exact_recv_per_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65707 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65731 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65419 expected_overflow=0 exact_recv_per_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65983 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65359 expected_overflow=0 exact_recv_per_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65376 expected_overflow=0 exact_recv_per_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65207 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65419 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65359 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65376 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65164 expected_overflow=0 exact_recv_per_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65429 expected_overflow=0 exact_recv_per_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65640 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65164 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65429 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65297 expected_overflow=0 exact_recv_per_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65901 expected_overflow=0 exact_recv_per_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65901 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=40960 expected_recv=66013 expected_overflow=0 exact_recv_per_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65297 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65437 expected_overflow=0 exact_recv_per_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=40960 expected_recv=66013 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65437 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65407 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65132 expected_overflow=0 exact_recv_per_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65506 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65930 expected_overflow=0 exact_recv_per_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65132 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65381 expected_overflow=0 exact_recv_per_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65930 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65752 expected_overflow=0 exact_recv_per_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65271 expected_overflow=0 exact_recv_per_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65728 expected_overflow=0 exact_recv_per_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65381 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65588 expected_overflow=0 exact_recv_per_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65752 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65271 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65728 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65588 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65293 expected_overflow=0 exact_recv_per_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65904 expected_overflow=0 exact_recv_per_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65293 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65904 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65722 expected_overflow=0 exact_recv_per_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65749 expected_overflow=0 exact_recv_per_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65722 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65120 expected_overflow=0 exact_recv_per_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65406 expected_overflow=0 exact_recv_per_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65716 expected_overflow=0 exact_recv_per_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65378 expected_overflow=0 exact_recv_per_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65749 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65120 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65406 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65716 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65378 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65839 expected_overflow=0 exact_recv_per_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65122 expected_overflow=0 exact_recv_per_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65839 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65122 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65980 expected_overflow=0 exact_recv_per_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65692 expected_overflow=0 exact_recv_per_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65980 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65692 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65320 expected_overflow=0 exact_recv_per_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65149 expected_overflow=0 exact_recv_per_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65586 expected_overflow=0 exact_recv_per_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65600 expected_overflow=0 exact_recv_per_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65320 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65586 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65149 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65600 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65783 expected_overflow=0 exact_recv_per_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65334 expected_overflow=0 exact_recv_per_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65540 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65783 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65334 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65437 expected_overflow=0 exact_recv_per_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65437 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65834 expected_overflow=0 exact_recv_per_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65457 expected_overflow=0 exact_recv_per_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65920 expected_overflow=0 exact_recv_per_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=40960 expected_recv=64983 expected_overflow=0 exact_recv_per_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65834 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65457 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65920 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=40960 expected_recv=64983 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65384 expected_overflow=0 exact_recv_per_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65630 expected_overflow=0 exact_recv_per_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65384 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65661 expected_overflow=0 exact_recv_per_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65630 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65661 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65731 expected_overflow=0 exact_recv_per_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65768 expected_overflow=0 exact_recv_per_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65231 expected_overflow=0 exact_recv_per_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65384 expected_overflow=0 exact_recv_per_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65731 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65499 expected_overflow=0 exact_recv_per_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65768 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65231 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65384 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65499 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65221 expected_overflow=0 exact_recv_per_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65239 expected_overflow=0 exact_recv_per_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65221 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65239 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65435 expected_overflow=0 exact_recv_per_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65435 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65969 expected_overflow=0 exact_recv_per_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65578 expected_overflow=0 exact_recv_per_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65888 expected_overflow=0 exact_recv_per_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65969 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65452 expected_overflow=0 exact_recv_per_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65578 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65888 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65620 expected_overflow=0 exact_recv_per_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65452 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65506 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65233 expected_overflow=0 exact_recv_per_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65620 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65470 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65233 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65612 expected_overflow=0 exact_recv_per_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65612 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65424 expected_overflow=0 exact_recv_per_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65571 expected_overflow=0 exact_recv_per_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65424 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65823 expected_overflow=0 exact_recv_per_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65377 expected_overflow=0 exact_recv_per_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65068 expected_overflow=0 exact_recv_per_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65571 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65823 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65377 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65068 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65616 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65379 expected_overflow=0 exact_recv_per_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65853 expected_overflow=0 exact_recv_per_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65379 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65780 expected_overflow=0 exact_recv_per_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65853 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65857 expected_overflow=0 exact_recv_per_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65780 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65494 expected_overflow=0 exact_recv_per_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65457 expected_overflow=0 exact_recv_per_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65857 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65281 expected_overflow=0 exact_recv_per_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65494 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65358 expected_overflow=0 exact_recv_per_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65457 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65281 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65358 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65787 expected_overflow=0 exact_recv_per_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65787 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65724 expected_overflow=0 exact_recv_per_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65491 expected_overflow=0 exact_recv_per_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65724 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65575 expected_overflow=0 exact_recv_per_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65491 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65575 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65803 expected_overflow=0 exact_recv_per_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65492 expected_overflow=0 exact_recv_per_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65570 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65479 expected_overflow=0 exact_recv_per_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65803 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65492 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65479 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65869 expected_overflow=0 exact_recv_per_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65869 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65439 expected_overflow=0 exact_recv_per_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65493 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65191 expected_overflow=0 exact_recv_per_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65439 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65388 expected_overflow=0 exact_recv_per_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65191 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=40960 expected_recv=66093 expected_overflow=0 exact_recv_per_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65674 expected_overflow=0 exact_recv_per_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65388 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65431 expected_overflow=0 exact_recv_per_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=40960 expected_recv=66093 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65674 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65431 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65629 expected_overflow=0 exact_recv_per_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65588 expected_overflow=0 exact_recv_per_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65629 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65588 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65040 expected_overflow=0 exact_recv_per_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65321 expected_overflow=0 exact_recv_per_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65401 expected_overflow=0 exact_recv_per_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65040 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65103 expected_overflow=0 exact_recv_per_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65321 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65401 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65364 expected_overflow=0 exact_recv_per_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65103 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65587 expected_overflow=0 exact_recv_per_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65364 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65555 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65587 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65517 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65790 expected_overflow=0 exact_recv_per_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65914 expected_overflow=0 exact_recv_per_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65654 expected_overflow=0 exact_recv_per_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65790 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65914 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65654 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65466 expected_overflow=0 exact_recv_per_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65612 expected_overflow=0 exact_recv_per_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65466 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65187 expected_overflow=0 exact_recv_per_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65612 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65623 expected_overflow=0 exact_recv_per_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65187 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65623 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65419 expected_overflow=0 exact_recv_per_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65993 expected_overflow=0 exact_recv_per_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65419 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65993 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65400 expected_overflow=0 exact_recv_per_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65400 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[32965, 32692, 33011, 32359, 32893, 32428, 32933, 32863] avg=32768.0000 max=33011 max_rank=2 min=32359 min_rank=3 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=12 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32485, 32786, 32660, 32827, 33137] avg=32768.0000 max=33137 max_rank=7 min=32485 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32485, 32786, 32660, 32827, 33137] avg=32768.0000 max=33137 max_rank=7 min=32485 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32485, 32786, 32660, 32827, 33137] avg=32768.0000 max=33137 max_rank=7 min=32485 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32485, 32786, 32660, 32827, 33137] avg=32768.0000 max=33137 max_rank=7 min=32485 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32485, 32786, 32660, 32827, 33137] avg=32768.0000 max=33137 max_rank=7 min=32485 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] avg=32768.0000 max=33096 max_rank=7 min=32526 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] avg=32768.0000 max=33096 max_rank=7 min=32526 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[32950, 32622, 32752, 32945, 32763, 32663, 32732, 32717] avg=32768.0000 max=32950 max_rank=0 min=32622 min_rank=1 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32485, 32786, 32660, 32827, 33137] avg=32768.0000 max=33137 max_rank=7 min=32485 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] avg=32768.0000 max=33096 max_rank=7 min=32526 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] avg=32768.0000 max=33096 max_rank=7 min=32526 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32485, 32786, 32660, 32827, 33137] avg=32768.0000 max=33137 max_rank=7 min=32485 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] avg=32768.0000 max=33096 max_rank=7 min=32526 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] avg=32768.0000 max=33096 max_rank=7 min=32526 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] avg=32768.0000 max=33096 max_rank=7 min=32526 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32485, 32786, 32660, 32827, 33137] avg=32768.0000 max=33137 max_rank=7 min=32485 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32674, 32852, 32723, 32526, 32786, 32660, 32827, 33096] avg=32768.0000 max=33096 max_rank=7 min=32526 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[33137, 32696, 32611, 32422, 32850, 32812, 32834, 32782] avg=32768.0000 max=33137 max_rank=0 min=32422 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[33137, 32696, 32611, 32422, 32850, 32812, 32834, 32782] avg=32768.0000 max=33137 max_rank=0 min=32422 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[33137, 32696, 32611, 32422, 32850, 32812, 32834, 32782] avg=32768.0000 max=33137 max_rank=0 min=32422 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[33137, 32696, 32611, 32422, 32850, 32812, 32834, 32782] avg=32768.0000 max=33137 max_rank=0 min=32422 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[33137, 32696, 32611, 32422, 32850, 32812, 32834, 32782] avg=32768.0000 max=33137 max_rank=0 min=32422 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[33137, 32696, 32611, 32422, 32850, 32812, 32834, 32782] avg=32768.0000 max=33137 max_rank=0 min=32422 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32612, 32885, 32724, 32809, 32941, 32950, 32804, 32419] avg=32768.0000 max=32950 max_rank=5 min=32419 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[33137, 32696, 32611, 32422, 32850, 32812, 32834, 32782] avg=32768.0000 max=33137 max_rank=0 min=32422 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32509, 32800, 32756, 32926, 33264, 32684, 32608] avg=32768.0000 max=33264 max_rank=5 min=32509 min_rank=1 imbalance_ratio=1.015137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32509, 32800, 32756, 32926, 33264, 32684, 32608] avg=32768.0000 max=33264 max_rank=5 min=32509 min_rank=1 imbalance_ratio=1.015137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[33137, 32696, 32611, 32422, 32850, 32812, 32834, 32782] avg=32768.0000 max=33137 max_rank=0 min=32422 min_rank=3 imbalance_ratio=1.011261 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32509, 32800, 32756, 32926, 33264, 32684, 32608] avg=32768.0000 max=33264 max_rank=5 min=32509 min_rank=1 imbalance_ratio=1.015137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32509, 32800, 32756, 32926, 33264, 32684, 32608] avg=32768.0000 max=33264 max_rank=5 min=32509 min_rank=1 imbalance_ratio=1.015137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32509, 32800, 32756, 32926, 33264, 32684, 32608] avg=32768.0000 max=33264 max_rank=5 min=32509 min_rank=1 imbalance_ratio=1.015137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=168 max_load_reduction_pct=0.005051 moved_to_replicas=168 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=168 max_load_reduction_pct=0.005051 moved_to_replicas=168 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32509, 32800, 32756, 32926, 33264, 32684, 32608] avg=32768.0000 max=33264 max_rank=5 min=32509 min_rank=1 imbalance_ratio=1.015137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[33096, 32696, 32611, 32463, 32850, 32812, 32834, 32782] avg=32768.0000 max=33096 max_rank=0 min=32463 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=41 max_load_reduction_pct=0.001237 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=168 max_load_reduction_pct=0.005051 moved_to_replicas=168 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=168 max_load_reduction_pct=0.005051 moved_to_replicas=168 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=168 max_load_reduction_pct=0.005051 moved_to_replicas=168 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32509, 32800, 32756, 32926, 33264, 32684, 32608] avg=32768.0000 max=33264 max_rank=5 min=32509 min_rank=1 imbalance_ratio=1.015137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=168 max_load_reduction_pct=0.005051 moved_to_replicas=168 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=168 max_load_reduction_pct=0.005051 moved_to_replicas=168 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32509, 32800, 32756, 32926, 33264, 32684, 32608] avg=32768.0000 max=33264 max_rank=5 min=32509 min_rank=1 imbalance_ratio=1.015137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32597, 32677, 32800, 32756, 32926, 33096, 32684, 32608] avg=32768.0000 max=33096 max_rank=5 min=32597 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=168 max_load_reduction_pct=0.005051 moved_to_replicas=168 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[32819, 32482, 32855, 32926, 32591, 32723, 32660, 33088] avg=32768.0000 max=33088 max_rank=7 min=32482 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32681, 32834, 32741, 32812, 32723, 32753, 32729, 32871] avg=32768.0000 max=32871 max_rank=7 min=32681 min_rank=0 imbalance_ratio=1.003143 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33098, 32619, 32575, 32901, 32814] avg=32768.0000 max=33098 max_rank=3 min=32575 min_rank=5 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33098, 32619, 32575, 32901, 32814] avg=32768.0000 max=33098 max_rank=3 min=32575 min_rank=5 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33098, 32619, 32575, 32901, 32814] avg=32768.0000 max=33098 max_rank=3 min=32575 min_rank=5 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33098, 32619, 32575, 32901, 32814] avg=32768.0000 max=33098 max_rank=3 min=32575 min_rank=5 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33098, 32619, 32575, 32901, 32814] avg=32768.0000 max=33098 max_rank=3 min=32575 min_rank=5 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] avg=32768.0000 max=33096 max_rank=3 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] avg=32768.0000 max=33096 max_rank=3 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33098, 32619, 32575, 32901, 32814] avg=32768.0000 max=33098 max_rank=3 min=32575 min_rank=5 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] avg=32768.0000 max=33096 max_rank=3 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] avg=32768.0000 max=33096 max_rank=3 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33098, 32619, 32575, 32901, 32814] avg=32768.0000 max=33098 max_rank=3 min=32575 min_rank=5 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] avg=32768.0000 max=33096 max_rank=3 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32583, 32753, 32686, 32586, 32996, 32830, 32919, 32791] avg=32768.0000 max=32996 max_rank=4 min=32583 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] avg=32768.0000 max=33096 max_rank=3 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] avg=32768.0000 max=33096 max_rank=3 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33098, 32619, 32575, 32901, 32814] avg=32768.0000 max=33098 max_rank=3 min=32575 min_rank=5 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32793, 32760, 32584, 33096, 32619, 32577, 32901, 32814] avg=32768.0000 max=33096 max_rank=3 min=32577 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[32582, 33018, 32637, 32725, 32829, 32787, 32667, 32899] avg=32768.0000 max=33018 max_rank=1 min=32582 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[32569, 32846, 32894, 32745, 32641, 32685, 32877, 32887] avg=32768.0000 max=32894 max_rank=2 min=32569 min_rank=0 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=88 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=88 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=88 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=88 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=88 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=88 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[32854, 32396, 32896, 32681, 32700, 32801, 33055, 32761] avg=32768.0000 max=33055 max_rank=6 min=32396 min_rank=1 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=88 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32530, 32780, 32997, 32839, 32364, 32979, 32979, 32676] avg=32768.0000 max=32997 max_rank=2 min=32364 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=88 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=50 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=50 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=50 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=50 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=50 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=50 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32813, 33006, 32556, 32801, 32790, 32777, 32793, 32608] avg=32768.0000 max=33006 max_rank=1 min=32556 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=50 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[32806, 32897, 32749, 32843, 32894, 32944, 32484, 32527] avg=32768.0000 max=32944 max_rank=5 min=32484 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=50 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=1048534 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254080 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=1049869 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252672 peak_demand=33074 required_capacity=1.009338 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=1049816 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253312 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=1047270 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255488 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=1048391 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254592 peak_demand=33000 required_capacity=1.007080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=1048478 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254336 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=1048285 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253824 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=1047965 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254976 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:12:54] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 814.4 | throughput per GPU (TFLOP/s/GPU): 409.4 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.328942E+01 | loss scale: 1.0 | grad norm: 27.108 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65581 expected_overflow=0 exact_recv_per_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65412 expected_overflow=0 exact_recv_per_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66059 expected_overflow=0 exact_recv_per_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65289 expected_overflow=0 exact_recv_per_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65698 expected_overflow=0 exact_recv_per_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65500 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65581 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65412 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66059 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65200 expected_overflow=0 exact_recv_per_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65289 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65698 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65549 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65200 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65480 expected_overflow=0 exact_recv_per_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65412 expected_overflow=0 exact_recv_per_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65477 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65725 expected_overflow=0 exact_recv_per_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65480 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65440 expected_overflow=0 exact_recv_per_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65725 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65483 expected_overflow=0 exact_recv_per_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65412 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65736 expected_overflow=0 exact_recv_per_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65440 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65483 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65736 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65445 expected_overflow=0 exact_recv_per_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65749 expected_overflow=0 exact_recv_per_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65675 expected_overflow=0 exact_recv_per_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65814 expected_overflow=0 exact_recv_per_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65445 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65749 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65342 expected_overflow=0 exact_recv_per_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65515 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65675 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65277 expected_overflow=0 exact_recv_per_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65814 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65342 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65471 expected_overflow=0 exact_recv_per_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65277 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65471 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65342 expected_overflow=0 exact_recv_per_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65467 expected_overflow=0 exact_recv_per_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65723 expected_overflow=0 exact_recv_per_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65445 expected_overflow=0 exact_recv_per_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65342 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65565 expected_overflow=0 exact_recv_per_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65467 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65723 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65745 expected_overflow=0 exact_recv_per_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65445 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65580 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65565 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65421 expected_overflow=0 exact_recv_per_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65745 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65421 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65850 expected_overflow=0 exact_recv_per_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65481 expected_overflow=0 exact_recv_per_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65424 expected_overflow=0 exact_recv_per_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65850 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65481 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65424 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65198 expected_overflow=0 exact_recv_per_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65489 expected_overflow=0 exact_recv_per_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65617 expected_overflow=0 exact_recv_per_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65198 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65489 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65627 expected_overflow=0 exact_recv_per_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65617 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65602 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65627 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65634 expected_overflow=0 exact_recv_per_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65684 expected_overflow=0 exact_recv_per_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65634 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65684 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65478 expected_overflow=0 exact_recv_per_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65478 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65638 expected_overflow=0 exact_recv_per_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65295 expected_overflow=0 exact_recv_per_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65494 expected_overflow=0 exact_recv_per_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65638 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65295 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65494 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65515 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65270 expected_overflow=0 exact_recv_per_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65529 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65270 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65549 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65239 expected_overflow=0 exact_recv_per_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65346 expected_overflow=0 exact_recv_per_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65456 expected_overflow=0 exact_recv_per_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65963 expected_overflow=0 exact_recv_per_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65239 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65936 expected_overflow=0 exact_recv_per_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65346 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65456 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65963 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65936 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65665 expected_overflow=0 exact_recv_per_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65353 expected_overflow=0 exact_recv_per_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65207 expected_overflow=0 exact_recv_per_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65665 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65353 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65894 expected_overflow=0 exact_recv_per_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65207 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65894 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65547 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65864 expected_overflow=0 exact_recv_per_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65474 expected_overflow=0 exact_recv_per_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65284 expected_overflow=0 exact_recv_per_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65864 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65474 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65284 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65424 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=40960 expected_recv=66109 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65424 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65381 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=40960 expected_recv=66109 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65652 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65381 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65676 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65652 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65431 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65080 expected_overflow=0 exact_recv_per_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65676 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65457 expected_overflow=0 exact_recv_per_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65431 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65080 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65841 expected_overflow=0 exact_recv_per_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65457 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65841 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65519 expected_overflow=0 exact_recv_per_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65519 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65289 expected_overflow=0 exact_recv_per_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65612 expected_overflow=0 exact_recv_per_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65289 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65293 expected_overflow=0 exact_recv_per_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65612 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65575 expected_overflow=0 exact_recv_per_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65702 expected_overflow=0 exact_recv_per_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65293 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65820 expected_overflow=0 exact_recv_per_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65468 expected_overflow=0 exact_recv_per_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65575 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65702 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65820 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65468 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65502 expected_overflow=0 exact_recv_per_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65537 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65502 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65185 expected_overflow=0 exact_recv_per_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65446 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65185 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65728 expected_overflow=0 exact_recv_per_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65745 expected_overflow=0 exact_recv_per_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65743 expected_overflow=0 exact_recv_per_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65585 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65728 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65745 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65743 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65499 expected_overflow=0 exact_recv_per_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65547 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65436 expected_overflow=0 exact_recv_per_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65499 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65106 expected_overflow=0 exact_recv_per_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65436 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65106 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65629 expected_overflow=0 exact_recv_per_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65453 expected_overflow=0 exact_recv_per_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65600 expected_overflow=0 exact_recv_per_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65629 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65694 expected_overflow=0 exact_recv_per_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65453 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65600 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65694 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65845 expected_overflow=0 exact_recv_per_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65573 expected_overflow=0 exact_recv_per_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65845 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65432 expected_overflow=0 exact_recv_per_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65200 expected_overflow=0 exact_recv_per_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65573 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65432 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65200 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65329 expected_overflow=0 exact_recv_per_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65652 expected_overflow=0 exact_recv_per_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65817 expected_overflow=0 exact_recv_per_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65762 expected_overflow=0 exact_recv_per_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65329 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65652 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65817 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65762 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65458 expected_overflow=0 exact_recv_per_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65199 expected_overflow=0 exact_recv_per_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65318 expected_overflow=0 exact_recv_per_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65458 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65199 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65447 expected_overflow=0 exact_recv_per_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65318 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65447 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65676 expected_overflow=0 exact_recv_per_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65502 expected_overflow=0 exact_recv_per_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65670 expected_overflow=0 exact_recv_per_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65721 expected_overflow=0 exact_recv_per_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65676 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65502 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65670 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65721 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65731 expected_overflow=0 exact_recv_per_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65363 expected_overflow=0 exact_recv_per_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65407 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65731 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65363 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65377 expected_overflow=0 exact_recv_per_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65514 expected_overflow=0 exact_recv_per_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65762 expected_overflow=0 exact_recv_per_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65377 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65680 expected_overflow=0 exact_recv_per_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65514 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65762 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65140 expected_overflow=0 exact_recv_per_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65680 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65631 expected_overflow=0 exact_recv_per_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65645 expected_overflow=0 exact_recv_per_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65140 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65631 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65645 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65586 expected_overflow=0 exact_recv_per_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65586 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65460 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[32934, 32672, 32764, 33001, 32770, 32560, 32886, 32557] avg=32768.0000 max=33001 max_rank=3 min=32557 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[32619, 32752, 32747, 32638, 32806, 32990, 32707, 32885] avg=32768.0000 max=32990 max_rank=5 min=32619 min_rank=0 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[32565, 32819, 32845, 32975, 32488, 32801, 32724, 32927] avg=32768.0000 max=32975 max_rank=3 min=32488 min_rank=4 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32907, 32850, 32648, 32676, 32877, 32688, 32764, 32734] avg=32768.0000 max=32907 max_rank=0 min=32648 min_rank=2 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32792, 32854, 32585, 32741, 32847, 32868, 32620, 32837] avg=32768.0000 max=32868 max_rank=5 min=32585 min_rank=2 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=14 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32689, 32801, 32763, 32707, 32703, 32527, 32881, 33073] avg=32768.0000 max=33073 max_rank=7 min=32527 min_rank=5 imbalance_ratio=1.009308 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=14 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=14 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=14 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=14 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=14 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=14 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[33020, 32818, 32594, 32872, 32903, 32840, 32526, 32571] avg=32768.0000 max=33020 max_rank=0 min=32526 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=14 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[33198, 32852, 32821, 32777, 32810, 32579, 32553, 32554] avg=32768.0000 max=33198 max_rank=0 min=32553 min_rank=6 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[33198, 32852, 32821, 32777, 32810, 32579, 32553, 32554] avg=32768.0000 max=33198 max_rank=0 min=32553 min_rank=6 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[33198, 32852, 32821, 32777, 32810, 32579, 32553, 32554] avg=32768.0000 max=33198 max_rank=0 min=32553 min_rank=6 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[33198, 32852, 32821, 32777, 32810, 32579, 32553, 32554] avg=32768.0000 max=33198 max_rank=0 min=32553 min_rank=6 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] avg=32768.0000 max=33096 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[33198, 32852, 32821, 32777, 32810, 32579, 32553, 32554] avg=32768.0000 max=33198 max_rank=0 min=32553 min_rank=6 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] avg=32768.0000 max=33096 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[33198, 32852, 32821, 32777, 32810, 32579, 32553, 32554] avg=32768.0000 max=33198 max_rank=0 min=32553 min_rank=6 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32587, 32756, 32901, 32704, 32780, 32656, 32903, 32857] avg=32768.0000 max=32903 max_rank=6 min=32587 min_rank=0 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] avg=32768.0000 max=33096 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] avg=32768.0000 max=33096 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] avg=32768.0000 max=33096 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] avg=32768.0000 max=33096 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[33198, 32852, 32821, 32777, 32810, 32579, 32553, 32554] avg=32768.0000 max=33198 max_rank=0 min=32553 min_rank=6 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[33198, 32852, 32821, 32777, 32810, 32579, 32553, 32554] avg=32768.0000 max=33198 max_rank=0 min=32553 min_rank=6 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] avg=32768.0000 max=33096 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32852, 32821, 32777, 32810, 32579, 32655, 32554] avg=32768.0000 max=33096 max_rank=0 min=32554 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32873, 32576, 32894, 32719, 32615, 32721, 32724, 33022] avg=32768.0000 max=33022 max_rank=7 min=32576 min_rank=1 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32638, 32802, 32707, 32742, 32678, 32999, 32671, 32907] avg=32768.0000 max=32999 max_rank=5 min=32638 min_rank=0 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32791, 32853, 32620, 32971, 32402, 32789, 32693, 33025] avg=32768.0000 max=33025 max_rank=7 min=32402 min_rank=4 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[32808, 32800, 32768, 32642, 32558, 32989, 32692, 32887] avg=32768.0000 max=32989 max_rank=5 min=32558 min_rank=4 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32867, 32840, 32493, 32767, 32634, 32968, 32666, 32909] avg=32768.0000 max=32968 max_rank=5 min=32493 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[32550, 32719, 32584, 32807, 32864, 32972, 32875, 32773] avg=32768.0000 max=32972 max_rank=5 min=32550 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[32614, 32876, 32797, 32819, 32793, 32656, 32790, 32799] avg=32768.0000 max=32876 max_rank=1 min=32614 min_rank=0 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=1047871 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254720 peak_demand=32991 required_capacity=1.006805 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=1049437 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252928 peak_demand=33073 required_capacity=1.009308 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=1048246 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254336 peak_demand=33084 required_capacity=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=1049103 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253056 peak_demand=33018 required_capacity=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=1047726 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254976 peak_demand=32901 required_capacity=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=1048405 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254208 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=1048880 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253952 peak_demand=33058 required_capacity=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=1048940 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254080 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:12:55] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 426.5 | throughput per GPU (TFLOP/s/GPU): 781.7 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.036214E+01 | loss scale: 1.0 | grad norm: 12.677 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65362 expected_overflow=0 exact_recv_per_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65755 expected_overflow=0 exact_recv_per_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65674 expected_overflow=0 exact_recv_per_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65464 expected_overflow=0 exact_recv_per_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65307 expected_overflow=0 exact_recv_per_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65362 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65556 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65627 expected_overflow=0 exact_recv_per_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65755 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65674 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65464 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65307 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=40960 expected_recv=65627 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65654 expected_overflow=0 exact_recv_per_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65641 expected_overflow=0 exact_recv_per_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65348 expected_overflow=0 exact_recv_per_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65654 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65679 expected_overflow=0 exact_recv_per_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65641 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65348 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65464 expected_overflow=0 exact_recv_per_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65679 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65524 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65621 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65357 expected_overflow=0 exact_recv_per_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65464 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65357 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65674 expected_overflow=0 exact_recv_per_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65156 expected_overflow=0 exact_recv_per_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65402 expected_overflow=0 exact_recv_per_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65674 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65402 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65925 expected_overflow=0 exact_recv_per_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65156 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65334 expected_overflow=0 exact_recv_per_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65790 expected_overflow=0 exact_recv_per_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65925 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65403 expected_overflow=0 exact_recv_per_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65604 expected_overflow=0 exact_recv_per_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65334 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65790 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65403 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65604 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65373 expected_overflow=0 exact_recv_per_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65373 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=40960 expected_recv=66074 expected_overflow=0 exact_recv_per_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=40960 expected_recv=66074 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65377 expected_overflow=0 exact_recv_per_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65684 expected_overflow=0 exact_recv_per_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65377 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=40960 expected_recv=64944 expected_overflow=0 exact_recv_per_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65709 expected_overflow=0 exact_recv_per_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65430 expected_overflow=0 exact_recv_per_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65684 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65602 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=40960 expected_recv=64944 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65709 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65430 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65348 expected_overflow=0 exact_recv_per_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65348 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65401 expected_overflow=0 exact_recv_per_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65817 expected_overflow=0 exact_recv_per_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65401 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65449 expected_overflow=0 exact_recv_per_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65720 expected_overflow=0 exact_recv_per_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65817 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65653 expected_overflow=0 exact_recv_per_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65634 expected_overflow=0 exact_recv_per_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65449 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65720 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65470 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65653 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65634 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65613 expected_overflow=0 exact_recv_per_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65814 expected_overflow=0 exact_recv_per_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65613 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65814 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65242 expected_overflow=0 exact_recv_per_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65534 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65488 expected_overflow=0 exact_recv_per_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65242 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65775 expected_overflow=0 exact_recv_per_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65459 expected_overflow=0 exact_recv_per_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65504 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65488 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65775 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65459 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65490 expected_overflow=0 exact_recv_per_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65597 expected_overflow=0 exact_recv_per_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65490 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65597 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65260 expected_overflow=0 exact_recv_per_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65435 expected_overflow=0 exact_recv_per_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65788 expected_overflow=0 exact_recv_per_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65334 expected_overflow=0 exact_recv_per_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65260 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65560 expected_overflow=0 exact_recv_per_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65609 expected_overflow=0 exact_recv_per_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65435 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65788 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65334 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65560 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65609 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65459 expected_overflow=0 exact_recv_per_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65940 expected_overflow=0 exact_recv_per_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65459 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65940 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65488 expected_overflow=0 exact_recv_per_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65597 expected_overflow=0 exact_recv_per_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65432 expected_overflow=0 exact_recv_per_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65571 expected_overflow=0 exact_recv_per_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65488 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65241 expected_overflow=0 exact_recv_per_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65597 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65432 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65571 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65605 expected_overflow=0 exact_recv_per_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65241 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65719 expected_overflow=0 exact_recv_per_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65605 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65719 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65671 expected_overflow=0 exact_recv_per_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65738 expected_overflow=0 exact_recv_per_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65455 expected_overflow=0 exact_recv_per_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65317 expected_overflow=0 exact_recv_per_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65664 expected_overflow=0 exact_recv_per_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65671 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65738 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65455 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65258 expected_overflow=0 exact_recv_per_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65317 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65664 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65258 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65518 expected_overflow=0 exact_recv_per_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65602 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65518 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65733 expected_overflow=0 exact_recv_per_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65688 expected_overflow=0 exact_recv_per_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65614 expected_overflow=0 exact_recv_per_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65751 expected_overflow=0 exact_recv_per_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65733 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65688 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65052 expected_overflow=0 exact_recv_per_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65614 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65751 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65280 expected_overflow=0 exact_recv_per_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65052 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65280 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65622 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65622 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65175 expected_overflow=0 exact_recv_per_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65629 expected_overflow=0 exact_recv_per_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65175 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65629 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65365 expected_overflow=0 exact_recv_per_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65602 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65702 expected_overflow=0 exact_recv_per_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65341 expected_overflow=0 exact_recv_per_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65492 expected_overflow=0 exact_recv_per_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65365 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65559 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65702 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65341 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65492 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65361 expected_overflow=0 exact_recv_per_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65422 expected_overflow=0 exact_recv_per_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65361 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65422 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65109 expected_overflow=0 exact_recv_per_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65663 expected_overflow=0 exact_recv_per_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65732 expected_overflow=0 exact_recv_per_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65554 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65771 expected_overflow=0 exact_recv_per_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65109 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65663 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65732 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65771 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65416 expected_overflow=0 exact_recv_per_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65852 expected_overflow=0 exact_recv_per_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65416 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65523 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65852 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65807 expected_overflow=0 exact_recv_per_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65581 expected_overflow=0 exact_recv_per_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65743 expected_overflow=0 exact_recv_per_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65497 expected_overflow=0 exact_recv_per_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65807 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65581 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65743 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65497 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65526 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65648 expected_overflow=0 exact_recv_per_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65470 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65524 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65648 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65362 expected_overflow=0 exact_recv_per_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65690 expected_overflow=0 exact_recv_per_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65362 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65412 expected_overflow=0 exact_recv_per_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65690 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65446 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65527 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65412 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65718 expected_overflow=0 exact_recv_per_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65434 expected_overflow=0 exact_recv_per_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65680 expected_overflow=0 exact_recv_per_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65718 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65434 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65680 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65448 expected_overflow=0 exact_recv_per_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65344 expected_overflow=0 exact_recv_per_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65528 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65476 expected_overflow=0 exact_recv_per_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65514 expected_overflow=0 exact_recv_per_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65448 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65344 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65476 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65514 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65380 expected_overflow=0 exact_recv_per_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65787 expected_overflow=0 exact_recv_per_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65380 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65787 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65675 expected_overflow=0 exact_recv_per_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65675 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65385 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65457 expected_overflow=0 exact_recv_per_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65457 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32533, 32844, 32624, 32912, 32972, 32685, 32707, 32867] avg=32768.0000 max=32972 max_rank=4 min=32533 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32703, 33079, 32766, 32670, 32730, 32769, 32709, 32718] avg=32768.0000 max=33079 max_rank=1 min=32670 min_rank=3 imbalance_ratio=1.009491 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32772, 32521, 32946, 32577, 32753, 33075, 32788, 32712] avg=32768.0000 max=33075 max_rank=5 min=32521 min_rank=1 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32442, 32995, 32836, 32840, 32864, 32568, 32825, 32774] avg=32768.0000 max=32995 max_rank=1 min=32442 min_rank=0 imbalance_ratio=1.006927 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[32784, 32722, 32766, 32709, 32679, 32946, 32820, 32718] avg=32768.0000 max=32946 max_rank=5 min=32679 min_rank=4 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32595, 33076, 32796, 32492, 32803, 32770, 32847, 32765] avg=32768.0000 max=33076 max_rank=1 min=32492 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32929, 32699, 32679, 32776, 32972, 32768, 32615, 32706] avg=32768.0000 max=32972 max_rank=4 min=32615 min_rank=6 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[32516, 32856, 32911, 32794, 32764, 32858, 32699, 32746] avg=32768.0000 max=32911 max_rank=2 min=32516 min_rank=0 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32669, 32971, 32682, 32689, 32833, 32875, 32641, 32784] avg=32768.0000 max=32971 max_rank=1 min=32641 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32696, 32579, 33034, 32582, 32765, 32987, 32698, 32803] avg=32768.0000 max=33034 max_rank=2 min=32579 min_rank=1 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[32742, 33029, 32865, 32832, 32712, 32494, 32636, 32834] avg=32768.0000 max=33029 max_rank=1 min=32494 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[33034, 32809, 32634, 32825, 32809, 32683, 32674, 32676] avg=32768.0000 max=33034 max_rank=0 min=32634 min_rank=2 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[32748, 32966, 32713, 32757, 32810, 32575, 32955, 32620] avg=32768.0000 max=32966 max_rank=1 min=32575 min_rank=5 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32745, 32857, 32877, 32761, 32591, 32759, 32732, 32822] avg=32768.0000 max=32877 max_rank=2 min=32591 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[32656, 32728, 32782, 32842, 32574, 32931, 32726, 32905] avg=32768.0000 max=32931 max_rank=5 min=32574 min_rank=4 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32733, 32734, 32775, 32867, 32806, 32671, 32755, 32803] avg=32768.0000 max=32867 max_rank=3 min=32671 min_rank=5 imbalance_ratio=1.003021 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=1049469 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253568 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=1049715 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252544 peak_demand=33084 required_capacity=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=1047938 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254208 peak_demand=32905 required_capacity=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=1049386 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253824 peak_demand=32981 required_capacity=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=1049103 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252800 peak_demand=33053 required_capacity=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=1048502 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254208 peak_demand=33075 required_capacity=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=1046970 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255744 peak_demand=33034 required_capacity=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=1047525 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255744 peak_demand=32912 required_capacity=1.004395 + [2026-08-30 05:12:55] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 422.1 | throughput per GPU (TFLOP/s/GPU): 789.8 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.048452E+01 | loss scale: 1.0 | grad norm: 44.588 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65795 expected_overflow=0 exact_recv_per_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65674 expected_overflow=0 exact_recv_per_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65431 expected_overflow=0 exact_recv_per_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65783 expected_overflow=0 exact_recv_per_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=40960 expected_recv=64977 expected_overflow=0 exact_recv_per_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65273 expected_overflow=0 exact_recv_per_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65795 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65321 expected_overflow=0 exact_recv_per_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65674 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66034 expected_overflow=0 exact_recv_per_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65431 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65783 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=40960 expected_recv=64977 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65273 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65321 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66034 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65791 expected_overflow=0 exact_recv_per_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65123 expected_overflow=0 exact_recv_per_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65791 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65123 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65686 expected_overflow=0 exact_recv_per_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65308 expected_overflow=0 exact_recv_per_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65686 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65224 expected_overflow=0 exact_recv_per_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65521 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65308 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65617 expected_overflow=0 exact_recv_per_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=40960 expected_recv=66018 expected_overflow=0 exact_recv_per_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65224 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65617 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=40960 expected_recv=66018 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65837 expected_overflow=0 exact_recv_per_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65579 expected_overflow=0 exact_recv_per_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65837 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65579 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65829 expected_overflow=0 exact_recv_per_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65024 expected_overflow=0 exact_recv_per_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65729 expected_overflow=0 exact_recv_per_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65609 expected_overflow=0 exact_recv_per_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65829 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65279 expected_overflow=0 exact_recv_per_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65024 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65729 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65402 expected_overflow=0 exact_recv_per_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65609 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65279 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65402 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65434 expected_overflow=0 exact_recv_per_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65584 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65434 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65296 expected_overflow=0 exact_recv_per_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65669 expected_overflow=0 exact_recv_per_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65336 expected_overflow=0 exact_recv_per_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65296 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65842 expected_overflow=0 exact_recv_per_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65669 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65555 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65336 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65842 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65263 expected_overflow=0 exact_recv_per_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65572 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65263 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65615 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65182 expected_overflow=0 exact_recv_per_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65756 expected_overflow=0 exact_recv_per_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65182 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65565 expected_overflow=0 exact_recv_per_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65756 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65565 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=40960 expected_recv=66008 expected_overflow=0 exact_recv_per_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65167 expected_overflow=0 exact_recv_per_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65732 expected_overflow=0 exact_recv_per_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=40960 expected_recv=66008 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65366 expected_overflow=0 exact_recv_per_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65637 expected_overflow=0 exact_recv_per_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65167 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65732 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65366 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65916 expected_overflow=0 exact_recv_per_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65637 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65360 expected_overflow=0 exact_recv_per_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65916 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65360 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65331 expected_overflow=0 exact_recv_per_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65741 expected_overflow=0 exact_recv_per_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65331 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65379 expected_overflow=0 exact_recv_per_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65741 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65891 expected_overflow=0 exact_recv_per_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65619 expected_overflow=0 exact_recv_per_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65379 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65181 expected_overflow=0 exact_recv_per_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65891 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65619 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65181 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65777 expected_overflow=0 exact_recv_per_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65777 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65294 expected_overflow=0 exact_recv_per_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65650 expected_overflow=0 exact_recv_per_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65294 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65271 expected_overflow=0 exact_recv_per_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65650 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65605 expected_overflow=0 exact_recv_per_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65271 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65465 expected_overflow=0 exact_recv_per_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65605 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65615 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65534 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65465 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65702 expected_overflow=0 exact_recv_per_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65702 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65307 expected_overflow=0 exact_recv_per_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65307 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65337 expected_overflow=0 exact_recv_per_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65534 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65794 expected_overflow=0 exact_recv_per_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65394 expected_overflow=0 exact_recv_per_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65337 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65450 expected_overflow=0 exact_recv_per_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65700 expected_overflow=0 exact_recv_per_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65394 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65794 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65372 expected_overflow=0 exact_recv_per_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65450 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65700 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65372 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65698 expected_overflow=0 exact_recv_per_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65698 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65802 expected_overflow=0 exact_recv_per_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65472 expected_overflow=0 exact_recv_per_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65400 expected_overflow=0 exact_recv_per_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65636 expected_overflow=0 exact_recv_per_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65391 expected_overflow=0 exact_recv_per_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65004 expected_overflow=0 exact_recv_per_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65802 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65472 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65400 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65561 expected_overflow=0 exact_recv_per_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65636 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65391 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65004 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65561 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65552 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65443 expected_overflow=0 exact_recv_per_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65715 expected_overflow=0 exact_recv_per_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65986 expected_overflow=0 exact_recv_per_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65482 expected_overflow=0 exact_recv_per_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65712 expected_overflow=0 exact_recv_per_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65443 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65715 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65628 expected_overflow=0 exact_recv_per_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65986 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65482 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65736 expected_overflow=0 exact_recv_per_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65712 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65628 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65736 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65633 expected_overflow=0 exact_recv_per_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65633 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65208 expected_overflow=0 exact_recv_per_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=40960 expected_recv=64855 expected_overflow=0 exact_recv_per_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65208 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=40960 expected_recv=64855 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65922 expected_overflow=0 exact_recv_per_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=40960 expected_recv=66034 expected_overflow=0 exact_recv_per_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65355 expected_overflow=0 exact_recv_per_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65440 expected_overflow=0 exact_recv_per_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65922 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=40960 expected_recv=66034 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65829 expected_overflow=0 exact_recv_per_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65355 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65440 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65678 expected_overflow=0 exact_recv_per_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65829 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65678 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65337 expected_overflow=0 exact_recv_per_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65337 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65349 expected_overflow=0 exact_recv_per_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65349 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65813 expected_overflow=0 exact_recv_per_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65378 expected_overflow=0 exact_recv_per_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65643 expected_overflow=0 exact_recv_per_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65919 expected_overflow=0 exact_recv_per_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65096 expected_overflow=0 exact_recv_per_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65813 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65378 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65643 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65919 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65096 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65640 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65210 expected_overflow=0 exact_recv_per_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65210 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65665 expected_overflow=0 exact_recv_per_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65665 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65738 expected_overflow=0 exact_recv_per_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65302 expected_overflow=0 exact_recv_per_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65630 expected_overflow=0 exact_recv_per_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65738 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65302 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65855 expected_overflow=0 exact_recv_per_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65496 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65630 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65385 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65855 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65039 expected_overflow=0 exact_recv_per_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65517 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65039 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65284 expected_overflow=0 exact_recv_per_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65628 expected_overflow=0 exact_recv_per_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65284 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65628 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65159 expected_overflow=0 exact_recv_per_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65647 expected_overflow=0 exact_recv_per_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65656 expected_overflow=0 exact_recv_per_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65159 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65647 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65761 expected_overflow=0 exact_recv_per_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65313 expected_overflow=0 exact_recv_per_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65656 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65761 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65313 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65818 expected_overflow=0 exact_recv_per_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65818 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65444 expected_overflow=0 exact_recv_per_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65650 expected_overflow=0 exact_recv_per_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65444 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65650 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65339 expected_overflow=0 exact_recv_per_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65708 expected_overflow=0 exact_recv_per_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65339 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65391 expected_overflow=0 exact_recv_per_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65708 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65607 expected_overflow=0 exact_recv_per_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65649 expected_overflow=0 exact_recv_per_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65391 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65582 expected_overflow=0 exact_recv_per_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65607 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65649 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65582 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65568 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=43 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=43 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=43 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=43 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=43 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=43 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=43 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[32678, 32699, 32904, 32755, 33031, 32872, 32411, 32794] avg=32768.0000 max=33031 max_rank=4 min=32411 min_rank=6 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=43 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=144 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32633, 32814, 32705, 32868, 32929, 32631, 32827, 32737] avg=32768.0000 max=32929 max_rank=4 min=32631 min_rank=5 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=144 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=144 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=144 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=144 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=144 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=144 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[32735, 33022, 32905, 32869, 32599, 32633, 32556, 32825] avg=32768.0000 max=33022 max_rank=1 min=32556 min_rank=6 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=144 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32609, 32835, 32650, 32796, 32954, 32902, 32557, 32841] avg=32768.0000 max=32954 max_rank=4 min=32557 min_rank=6 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[33094, 32497, 32882, 32902, 32416, 32710, 32599, 33044] avg=32768.0000 max=33094 max_rank=0 min=32416 min_rank=4 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[32832, 32774, 32849, 32584, 32595, 32703, 32977, 32830] avg=32768.0000 max=32977 max_rank=6 min=32584 min_rank=3 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32746, 32848, 32951, 32919, 32470, 32747, 32616, 32847] avg=32768.0000 max=32951 max_rank=2 min=32470 min_rank=4 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32823, 32891, 32663, 32901, 32697, 32443, 32805, 32921] avg=32768.0000 max=32921 max_rank=7 min=32443 min_rank=5 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[32560, 32561, 32768, 32819, 32949, 32859, 32924, 32704] avg=32768.0000 max=32949 max_rank=4 min=32560 min_rank=0 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32890, 32748, 32774, 32758, 32783, 32762, 32552, 32877] avg=32768.0000 max=32890 max_rank=0 min=32552 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=41 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[33016, 32715, 32909, 32893, 32506, 32738, 32853, 32514] avg=32768.0000 max=33016 max_rank=0 min=32506 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[32893, 32653, 32857, 32957, 32752, 32729, 32819, 32484] avg=32768.0000 max=32957 max_rank=3 min=32484 min_rank=7 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[32758, 32746, 32906, 32475, 32641, 32798, 33055, 32765] avg=32768.0000 max=33055 max_rank=6 min=32475 min_rank=3 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[32971, 32819, 32774, 32613, 32874, 32934, 32563, 32596] avg=32768.0000 max=32971 max_rank=0 min=32563 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[32828, 32757, 32564, 32625, 32689, 32944, 32880, 32857] avg=32768.0000 max=32944 max_rank=5 min=32564 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[32877, 32758, 32769, 32689, 32646, 32838, 32833, 32734] avg=32768.0000 max=32877 max_rank=0 min=32646 min_rank=4 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=1048739 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254464 peak_demand=33065 required_capacity=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=1047238 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255488 peak_demand=33067 required_capacity=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=1048876 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253824 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=1047806 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254464 peak_demand=33089 required_capacity=1.009796 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=1048024 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254080 peak_demand=33044 required_capacity=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=1049296 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253312 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=1049236 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252800 peak_demand=33031 required_capacity=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=1049393 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252672 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:12:56] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 426.5 | throughput per GPU (TFLOP/s/GPU): 781.7 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.951463E+00 | loss scale: 1.0 | grad norm: 9.188 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65424 expected_overflow=0 exact_recv_per_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65026 expected_overflow=0 exact_recv_per_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65704 expected_overflow=0 exact_recv_per_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65789 expected_overflow=0 exact_recv_per_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65818 expected_overflow=0 exact_recv_per_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65234 expected_overflow=0 exact_recv_per_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65789 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65424 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65026 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65615 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65704 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65678 expected_overflow=0 exact_recv_per_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65818 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65234 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65678 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65586 expected_overflow=0 exact_recv_per_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65683 expected_overflow=0 exact_recv_per_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65168 expected_overflow=0 exact_recv_per_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65513 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65586 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65622 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65683 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65628 expected_overflow=0 exact_recv_per_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65168 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65508 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65628 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65580 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65166 expected_overflow=0 exact_recv_per_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65562 expected_overflow=0 exact_recv_per_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65780 expected_overflow=0 exact_recv_per_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65674 expected_overflow=0 exact_recv_per_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65166 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65562 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65471 expected_overflow=0 exact_recv_per_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65780 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65674 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65511 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65353 expected_overflow=0 exact_recv_per_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65471 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65771 expected_overflow=0 exact_recv_per_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65353 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65771 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65121 expected_overflow=0 exact_recv_per_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65816 expected_overflow=0 exact_recv_per_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65121 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65816 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65611 expected_overflow=0 exact_recv_per_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65264 expected_overflow=0 exact_recv_per_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65674 expected_overflow=0 exact_recv_per_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65611 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65574 expected_overflow=0 exact_recv_per_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65264 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65670 expected_overflow=0 exact_recv_per_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65674 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65574 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65670 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65453 expected_overflow=0 exact_recv_per_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=40960 expected_recv=64939 expected_overflow=0 exact_recv_per_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65656 expected_overflow=0 exact_recv_per_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65453 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=40960 expected_recv=64939 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65656 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65824 expected_overflow=0 exact_recv_per_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65789 expected_overflow=0 exact_recv_per_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65507 expected_overflow=0 exact_recv_per_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65500 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65620 expected_overflow=0 exact_recv_per_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65824 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65789 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65507 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65620 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65837 expected_overflow=0 exact_recv_per_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65261 expected_overflow=0 exact_recv_per_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65837 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65261 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65598 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65656 expected_overflow=0 exact_recv_per_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65326 expected_overflow=0 exact_recv_per_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65353 expected_overflow=0 exact_recv_per_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65572 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65685 expected_overflow=0 exact_recv_per_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65656 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65326 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65353 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65685 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65784 expected_overflow=0 exact_recv_per_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65263 expected_overflow=0 exact_recv_per_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65784 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65263 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65802 expected_overflow=0 exact_recv_per_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65871 expected_overflow=0 exact_recv_per_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65802 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65282 expected_overflow=0 exact_recv_per_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65703 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65077 expected_overflow=0 exact_recv_per_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65871 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65506 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65282 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65077 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65228 expected_overflow=0 exact_recv_per_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65647 expected_overflow=0 exact_recv_per_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65228 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65647 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65620 expected_overflow=0 exact_recv_per_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65440 expected_overflow=0 exact_recv_per_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65333 expected_overflow=0 exact_recv_per_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65620 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65632 expected_overflow=0 exact_recv_per_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65843 expected_overflow=0 exact_recv_per_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65333 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65440 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65545 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65632 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65843 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65564 expected_overflow=0 exact_recv_per_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65564 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65640 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65340 expected_overflow=0 exact_recv_per_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65635 expected_overflow=0 exact_recv_per_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65608 expected_overflow=0 exact_recv_per_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65409 expected_overflow=0 exact_recv_per_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65555 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65340 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65635 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65608 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65409 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65537 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65133 expected_overflow=0 exact_recv_per_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65133 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65477 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65437 expected_overflow=0 exact_recv_per_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65749 expected_overflow=0 exact_recv_per_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65713 expected_overflow=0 exact_recv_per_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65684 expected_overflow=0 exact_recv_per_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65765 expected_overflow=0 exact_recv_per_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65437 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65330 expected_overflow=0 exact_recv_per_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65749 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65713 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65684 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65765 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65330 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65253 expected_overflow=0 exact_recv_per_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65253 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65515 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65319 expected_overflow=0 exact_recv_per_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65319 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65774 expected_overflow=0 exact_recv_per_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65602 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65510 expected_overflow=0 exact_recv_per_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65718 expected_overflow=0 exact_recv_per_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65774 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65597 expected_overflow=0 exact_recv_per_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65510 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65718 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65405 expected_overflow=0 exact_recv_per_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65597 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65202 expected_overflow=0 exact_recv_per_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65405 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65202 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65762 expected_overflow=0 exact_recv_per_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=40960 expected_recv=64982 expected_overflow=0 exact_recv_per_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65762 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65635 expected_overflow=0 exact_recv_per_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=40960 expected_recv=64982 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65981 expected_overflow=0 exact_recv_per_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65630 expected_overflow=0 exact_recv_per_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65635 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65691 expected_overflow=0 exact_recv_per_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65386 expected_overflow=0 exact_recv_per_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65981 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65630 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65691 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65516 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65386 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65651 expected_overflow=0 exact_recv_per_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65651 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65878 expected_overflow=0 exact_recv_per_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65464 expected_overflow=0 exact_recv_per_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65878 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65140 expected_overflow=0 exact_recv_per_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65464 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65654 expected_overflow=0 exact_recv_per_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65599 expected_overflow=0 exact_recv_per_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65617 expected_overflow=0 exact_recv_per_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65140 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65654 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65599 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65617 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65407 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65262 expected_overflow=0 exact_recv_per_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65409 expected_overflow=0 exact_recv_per_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65262 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65409 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65799 expected_overflow=0 exact_recv_per_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65835 expected_overflow=0 exact_recv_per_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65401 expected_overflow=0 exact_recv_per_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65799 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65251 expected_overflow=0 exact_recv_per_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65835 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65401 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65251 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65548 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65514 expected_overflow=0 exact_recv_per_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65691 expected_overflow=0 exact_recv_per_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65514 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65739 expected_overflow=0 exact_recv_per_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65691 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65739 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65372 expected_overflow=0 exact_recv_per_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65742 expected_overflow=0 exact_recv_per_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65372 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=40960 expected_recv=66011 expected_overflow=0 exact_recv_per_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65431 expected_overflow=0 exact_recv_per_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65204 expected_overflow=0 exact_recv_per_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65742 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=40960 expected_recv=66011 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65431 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65633 expected_overflow=0 exact_recv_per_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65204 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65633 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65396 expected_overflow=0 exact_recv_per_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65831 expected_overflow=0 exact_recv_per_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65396 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65831 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65354 expected_overflow=0 exact_recv_per_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=40960 expected_recv=64939 expected_overflow=0 exact_recv_per_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65354 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65920 expected_overflow=0 exact_recv_per_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32517, 32804, 32792, 33163, 32595] avg=32768.0000 max=33163 max_rank=6 min=32517 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=40960 expected_recv=64939 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65920 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32517, 32804, 32792, 33163, 32595] avg=32768.0000 max=33163 max_rank=6 min=32517 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32517, 32804, 32792, 33163, 32595] avg=32768.0000 max=33163 max_rank=6 min=32517 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] avg=32768.0000 max=33096 max_rank=6 min=32545 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32517, 32804, 32792, 33163, 32595] avg=32768.0000 max=33163 max_rank=6 min=32517 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32517, 32804, 32792, 33163, 32595] avg=32768.0000 max=33163 max_rank=6 min=32517 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] avg=32768.0000 max=33096 max_rank=6 min=32545 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] avg=32768.0000 max=33096 max_rank=6 min=32545 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32517, 32804, 32792, 33163, 32595] avg=32768.0000 max=33163 max_rank=6 min=32517 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32517, 32804, 32792, 33163, 32595] avg=32768.0000 max=33163 max_rank=6 min=32517 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32517, 32804, 32792, 33163, 32595] avg=32768.0000 max=33163 max_rank=6 min=32517 min_rank=3 imbalance_ratio=1.012054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] avg=32768.0000 max=33096 max_rank=6 min=32545 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] avg=32768.0000 max=33096 max_rank=6 min=32545 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] avg=32768.0000 max=33096 max_rank=6 min=32545 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] avg=32768.0000 max=33096 max_rank=6 min=32545 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32545, 32836, 32892, 32584, 32804, 32792, 33096, 32595] avg=32768.0000 max=33096 max_rank=6 min=32545 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67 max_load_reduction_pct=0.002020 moved_to_replicas=67 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32770, 32761, 32661, 32846, 32755, 32717, 32803, 32831] avg=32768.0000 max=32846 max_rank=3 min=32661 min_rank=2 imbalance_ratio=1.002380 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32605, 32703, 32908, 32857, 32879, 32796, 32580, 32816] avg=32768.0000 max=32908 max_rank=2 min=32580 min_rank=6 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32557, 32863, 32699, 32908, 32969, 32622, 32689, 32837] avg=32768.0000 max=32969 max_rank=4 min=32557 min_rank=0 imbalance_ratio=1.006134 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[33157, 32550, 32705, 32815, 32940, 32634, 32815, 32528] avg=32768.0000 max=33157 max_rank=0 min=32528 min_rank=7 imbalance_ratio=1.011871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] avg=32768.0000 max=33096 max_rank=0 min=32550 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61 max_load_reduction_pct=0.001840 moved_to_replicas=61 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[33157, 32550, 32705, 32815, 32940, 32634, 32815, 32528] avg=32768.0000 max=33157 max_rank=0 min=32528 min_rank=7 imbalance_ratio=1.011871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[33157, 32550, 32705, 32815, 32940, 32634, 32815, 32528] avg=32768.0000 max=33157 max_rank=0 min=32528 min_rank=7 imbalance_ratio=1.011871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[33157, 32550, 32705, 32815, 32940, 32634, 32815, 32528] avg=32768.0000 max=33157 max_rank=0 min=32528 min_rank=7 imbalance_ratio=1.011871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[33157, 32550, 32705, 32815, 32940, 32634, 32815, 32528] avg=32768.0000 max=33157 max_rank=0 min=32528 min_rank=7 imbalance_ratio=1.011871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32821, 32662, 32685, 32717, 32594, 33085, 32718, 32862] avg=32768.0000 max=33085 max_rank=5 min=32594 min_rank=4 imbalance_ratio=1.009674 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[33157, 32550, 32705, 32815, 32940, 32634, 32815, 32528] avg=32768.0000 max=33157 max_rank=0 min=32528 min_rank=7 imbalance_ratio=1.011871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] avg=32768.0000 max=33096 max_rank=0 min=32550 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61 max_load_reduction_pct=0.001840 moved_to_replicas=61 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] avg=32768.0000 max=33096 max_rank=0 min=32550 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61 max_load_reduction_pct=0.001840 moved_to_replicas=61 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[33157, 32550, 32705, 32815, 32940, 32634, 32815, 32528] avg=32768.0000 max=33157 max_rank=0 min=32528 min_rank=7 imbalance_ratio=1.011871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] avg=32768.0000 max=33096 max_rank=0 min=32550 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61 max_load_reduction_pct=0.001840 moved_to_replicas=61 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] avg=32768.0000 max=33096 max_rank=0 min=32550 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61 max_load_reduction_pct=0.001840 moved_to_replicas=61 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33191, 32462, 32637, 32964, 32664] avg=32768.0000 max=33191 max_rank=3 min=32462 min_rank=4 imbalance_ratio=1.012909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] avg=32768.0000 max=33096 max_rank=0 min=32550 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61 max_load_reduction_pct=0.001840 moved_to_replicas=61 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[33157, 32550, 32705, 32815, 32940, 32634, 32815, 32528] avg=32768.0000 max=33157 max_rank=0 min=32528 min_rank=7 imbalance_ratio=1.011871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] avg=32768.0000 max=33096 max_rank=0 min=32550 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61 max_load_reduction_pct=0.001840 moved_to_replicas=61 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33191, 32462, 32637, 32964, 32664] avg=32768.0000 max=33191 max_rank=3 min=32462 min_rank=4 imbalance_ratio=1.012909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33191, 32462, 32637, 32964, 32664] avg=32768.0000 max=33191 max_rank=3 min=32462 min_rank=4 imbalance_ratio=1.012909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] avg=32768.0000 max=33096 max_rank=3 min=32557 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95 max_load_reduction_pct=0.002862 moved_to_replicas=95 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33191, 32462, 32637, 32964, 32664] avg=32768.0000 max=33191 max_rank=3 min=32462 min_rank=4 imbalance_ratio=1.012909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33191, 32462, 32637, 32964, 32664] avg=32768.0000 max=33191 max_rank=3 min=32462 min_rank=4 imbalance_ratio=1.012909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32550, 32705, 32815, 32940, 32634, 32815, 32589] avg=32768.0000 max=33096 max_rank=0 min=32550 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61 max_load_reduction_pct=0.001840 moved_to_replicas=61 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33191, 32462, 32637, 32964, 32664] avg=32768.0000 max=33191 max_rank=3 min=32462 min_rank=4 imbalance_ratio=1.012909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] avg=32768.0000 max=33096 max_rank=3 min=32557 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95 max_load_reduction_pct=0.002862 moved_to_replicas=95 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] avg=32768.0000 max=33096 max_rank=3 min=32557 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95 max_load_reduction_pct=0.002862 moved_to_replicas=95 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] avg=32768.0000 max=33096 max_rank=3 min=32557 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95 max_load_reduction_pct=0.002862 moved_to_replicas=95 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33191, 32462, 32637, 32964, 32664] avg=32768.0000 max=33191 max_rank=3 min=32462 min_rank=4 imbalance_ratio=1.012909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] avg=32768.0000 max=33096 max_rank=3 min=32557 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95 max_load_reduction_pct=0.002862 moved_to_replicas=95 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] avg=32768.0000 max=33096 max_rank=3 min=32557 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95 max_load_reduction_pct=0.002862 moved_to_replicas=95 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33191, 32462, 32637, 32964, 32664] avg=32768.0000 max=33191 max_rank=3 min=32462 min_rank=4 imbalance_ratio=1.012909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] avg=32768.0000 max=33096 max_rank=3 min=32557 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95 max_load_reduction_pct=0.002862 moved_to_replicas=95 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32575, 32843, 32808, 33096, 32557, 32637, 32964, 32664] avg=32768.0000 max=33096 max_rank=3 min=32557 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95 max_load_reduction_pct=0.002862 moved_to_replicas=95 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32712, 32510, 32680, 32777, 32890, 32998, 32612, 32965] avg=32768.0000 max=32998 max_rank=5 min=32510 min_rank=1 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33218, 32599, 32981, 32353, 32910, 32751] avg=32768.0000 max=33218 max_rank=2 min=32353 min_rank=5 imbalance_ratio=1.013733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33218, 32599, 32981, 32353, 32910, 32751] avg=32768.0000 max=33218 max_rank=2 min=32353 min_rank=5 imbalance_ratio=1.013733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33218, 32599, 32981, 32353, 32910, 32751] avg=32768.0000 max=33218 max_rank=2 min=32353 min_rank=5 imbalance_ratio=1.013733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] avg=32768.0000 max=33096 max_rank=2 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=122 max_load_reduction_pct=0.003673 moved_to_replicas=122 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33218, 32599, 32981, 32353, 32910, 32751] avg=32768.0000 max=33218 max_rank=2 min=32353 min_rank=5 imbalance_ratio=1.013733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33218, 32599, 32981, 32353, 32910, 32751] avg=32768.0000 max=33218 max_rank=2 min=32353 min_rank=5 imbalance_ratio=1.013733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32793, 32794, 32544, 32738, 32712, 33010, 32712, 32841] avg=32768.0000 max=33010 max_rank=5 min=32544 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] avg=32768.0000 max=33096 max_rank=2 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=122 max_load_reduction_pct=0.003673 moved_to_replicas=122 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] avg=32768.0000 max=33096 max_rank=2 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=122 max_load_reduction_pct=0.003673 moved_to_replicas=122 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33218, 32599, 32981, 32353, 32910, 32751] avg=32768.0000 max=33218 max_rank=2 min=32353 min_rank=5 imbalance_ratio=1.013733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] avg=32768.0000 max=33096 max_rank=2 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=122 max_load_reduction_pct=0.003673 moved_to_replicas=122 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] avg=32768.0000 max=33096 max_rank=2 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=122 max_load_reduction_pct=0.003673 moved_to_replicas=122 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33218, 32599, 32981, 32353, 32910, 32751] avg=32768.0000 max=33218 max_rank=2 min=32353 min_rank=5 imbalance_ratio=1.013733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] avg=32768.0000 max=33096 max_rank=2 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=122 max_load_reduction_pct=0.003673 moved_to_replicas=122 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33218, 32599, 32981, 32353, 32910, 32751] avg=32768.0000 max=33218 max_rank=2 min=32353 min_rank=5 imbalance_ratio=1.013733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] avg=32768.0000 max=33096 max_rank=2 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=122 max_load_reduction_pct=0.003673 moved_to_replicas=122 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=4 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32807, 32525, 33096, 32599, 32981, 32475, 32910, 32751] avg=32768.0000 max=33096 max_rank=2 min=32475 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=122 max_load_reduction_pct=0.003673 moved_to_replicas=122 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=4 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=4 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=4 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=4 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=4 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=4 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[32738, 32634, 32878, 32873, 32785, 32853, 32705, 32678] avg=32768.0000 max=32878 max_rank=2 min=32634 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=4 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[32706, 32990, 32426, 32929, 32725, 32780, 32702, 32886] avg=32768.0000 max=32990 max_rank=1 min=32426 min_rank=2 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32738, 32788, 32933, 32480, 32990, 32694, 32837, 32684] avg=32768.0000 max=32990 max_rank=4 min=32480 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[32753, 32748, 32508, 33056, 32792, 32651, 32801, 32835] avg=32768.0000 max=33056 max_rank=3 min=32508 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32711, 32593, 32893, 32854, 32890, 32640, 32768, 32795] avg=32768.0000 max=32893 max_rank=2 min=32593 min_rank=1 imbalance_ratio=1.003815 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[32534, 33009, 32745, 32734, 32570, 32911, 32802, 32839] avg=32768.0000 max=33009 max_rank=1 min=32534 min_rank=0 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=1048649 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253056 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=1047715 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254848 peak_demand=33009 required_capacity=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=1049549 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253056 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=1049616 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253184 peak_demand=33043 required_capacity=1.008392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=1048567 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254208 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=1048842 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253440 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=1046803 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=256000 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=1048867 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253952 peak_demand=33085 required_capacity=1.009674 + [2026-08-30 05:12:56] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 426.8 | throughput per GPU (TFLOP/s/GPU): 781.1 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.482718E+00 | loss scale: 1.0 | grad norm: 5.604 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65854 expected_overflow=0 exact_recv_per_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65645 expected_overflow=0 exact_recv_per_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65257 expected_overflow=0 exact_recv_per_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65454 expected_overflow=0 exact_recv_per_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65626 expected_overflow=0 exact_recv_per_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65854 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65531 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65645 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65385 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65257 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65454 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65626 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65695 expected_overflow=0 exact_recv_per_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65579 expected_overflow=0 exact_recv_per_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65487 expected_overflow=0 exact_recv_per_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65825 expected_overflow=0 exact_recv_per_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65695 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65487 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65579 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65289 expected_overflow=0 exact_recv_per_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65594 expected_overflow=0 exact_recv_per_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65825 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65289 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65277 expected_overflow=0 exact_recv_per_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65594 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65277 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65751 expected_overflow=0 exact_recv_per_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65651 expected_overflow=0 exact_recv_per_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65751 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65570 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65651 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65769 expected_overflow=0 exact_recv_per_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65462 expected_overflow=0 exact_recv_per_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65319 expected_overflow=0 exact_recv_per_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65383 expected_overflow=0 exact_recv_per_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65769 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65383 expected_overflow=0 exact_recv_per_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65462 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65319 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65383 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65383 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65945 expected_overflow=0 exact_recv_per_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65945 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65185 expected_overflow=0 exact_recv_per_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65460 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65185 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65364 expected_overflow=0 exact_recv_per_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65343 expected_overflow=0 exact_recv_per_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65670 expected_overflow=0 exact_recv_per_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65786 expected_overflow=0 exact_recv_per_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65364 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65343 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65670 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65786 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65582 expected_overflow=0 exact_recv_per_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65331 expected_overflow=0 exact_recv_per_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65108 expected_overflow=0 exact_recv_per_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65582 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65331 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65108 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65393 expected_overflow=0 exact_recv_per_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65663 expected_overflow=0 exact_recv_per_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65745 expected_overflow=0 exact_recv_per_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65639 expected_overflow=0 exact_recv_per_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65827 expected_overflow=0 exact_recv_per_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65393 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65663 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65745 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65639 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65827 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65572 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65423 expected_overflow=0 exact_recv_per_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65547 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65423 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65381 expected_overflow=0 exact_recv_per_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65677 expected_overflow=0 exact_recv_per_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65624 expected_overflow=0 exact_recv_per_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65381 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65361 expected_overflow=0 exact_recv_per_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65677 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65624 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65703 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65361 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65659 expected_overflow=0 exact_recv_per_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65810 expected_overflow=0 exact_recv_per_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65563 expected_overflow=0 exact_recv_per_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65659 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65810 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65563 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65438 expected_overflow=0 exact_recv_per_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65416 expected_overflow=0 exact_recv_per_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65206 expected_overflow=0 exact_recv_per_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65654 expected_overflow=0 exact_recv_per_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65438 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65416 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65206 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65654 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65251 expected_overflow=0 exact_recv_per_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65387 expected_overflow=0 exact_recv_per_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65953 expected_overflow=0 exact_recv_per_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65251 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65387 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65953 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65362 expected_overflow=0 exact_recv_per_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65687 expected_overflow=0 exact_recv_per_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65214 expected_overflow=0 exact_recv_per_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65864 expected_overflow=0 exact_recv_per_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65362 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65687 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65214 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65570 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65864 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65333 expected_overflow=0 exact_recv_per_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65315 expected_overflow=0 exact_recv_per_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65400 expected_overflow=0 exact_recv_per_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65333 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65315 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65400 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65876 expected_overflow=0 exact_recv_per_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65662 expected_overflow=0 exact_recv_per_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65658 expected_overflow=0 exact_recv_per_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65876 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65505 expected_overflow=0 exact_recv_per_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65662 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65658 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65505 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65749 expected_overflow=0 exact_recv_per_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65115 expected_overflow=0 exact_recv_per_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65265 expected_overflow=0 exact_recv_per_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65749 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65115 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65265 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65664 expected_overflow=0 exact_recv_per_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65709 expected_overflow=0 exact_recv_per_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65289 expected_overflow=0 exact_recv_per_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65731 expected_overflow=0 exact_recv_per_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65664 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65709 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65289 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65766 expected_overflow=0 exact_recv_per_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65731 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65766 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65623 expected_overflow=0 exact_recv_per_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65698 expected_overflow=0 exact_recv_per_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65623 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65698 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65338 expected_overflow=0 exact_recv_per_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65934 expected_overflow=0 exact_recv_per_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65283 expected_overflow=0 exact_recv_per_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65261 expected_overflow=0 exact_recv_per_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65601 expected_overflow=0 exact_recv_per_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65338 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65934 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65283 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65261 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65193 expected_overflow=0 exact_recv_per_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65601 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65041 expected_overflow=0 exact_recv_per_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65409 expected_overflow=0 exact_recv_per_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65193 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65041 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65409 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65804 expected_overflow=0 exact_recv_per_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65896 expected_overflow=0 exact_recv_per_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65583 expected_overflow=0 exact_recv_per_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65773 expected_overflow=0 exact_recv_per_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65804 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65589 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65896 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65583 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65773 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65688 expected_overflow=0 exact_recv_per_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65707 expected_overflow=0 exact_recv_per_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65446 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65688 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65707 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65752 expected_overflow=0 exact_recv_per_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65487 expected_overflow=0 exact_recv_per_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=40960 expected_recv=64962 expected_overflow=0 exact_recv_per_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65630 expected_overflow=0 exact_recv_per_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65752 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65487 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=40960 expected_recv=64962 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65630 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65337 expected_overflow=0 exact_recv_per_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65616 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65715 expected_overflow=0 exact_recv_per_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65271 expected_overflow=0 exact_recv_per_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65337 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65715 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65271 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65394 expected_overflow=0 exact_recv_per_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65499 expected_overflow=0 exact_recv_per_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65416 expected_overflow=0 exact_recv_per_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65394 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65499 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65416 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65703 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65953 expected_overflow=0 exact_recv_per_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65610 expected_overflow=0 exact_recv_per_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65845 expected_overflow=0 exact_recv_per_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65953 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65610 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65567 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65845 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65235 expected_overflow=0 exact_recv_per_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65411 expected_overflow=0 exact_recv_per_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65487 expected_overflow=0 exact_recv_per_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65235 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65411 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65487 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65501 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65230 expected_overflow=0 exact_recv_per_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65632 expected_overflow=0 exact_recv_per_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65320 expected_overflow=0 exact_recv_per_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65760 expected_overflow=0 exact_recv_per_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65230 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65632 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65320 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65760 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65472 expected_overflow=0 exact_recv_per_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65451 expected_overflow=0 exact_recv_per_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65842 expected_overflow=0 exact_recv_per_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65446 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65472 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65451 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65842 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65767 expected_overflow=0 exact_recv_per_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65767 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32830, 32935, 32674, 32836, 32751, 32621, 32910, 32587] avg=32768.0000 max=32935 max_rank=1 min=32587 min_rank=7 imbalance_ratio=1.005096 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32781, 32655, 32916, 32934, 32700, 32739, 32688, 32731] avg=32768.0000 max=32934 max_rank=3 min=32655 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32674, 32849, 32755, 32843, 32762, 32834, 32653, 32774] avg=32768.0000 max=32849 max_rank=1 min=32653 min_rank=6 imbalance_ratio=1.002472 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32427, 32817, 32705, 32799, 32989, 33138, 32644] avg=32768.0000 max=33138 max_rank=6 min=32427 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32427, 32817, 32705, 32799, 32989, 33138, 32644] avg=32768.0000 max=33138 max_rank=6 min=32427 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32427, 32817, 32705, 32799, 32989, 33138, 32644] avg=32768.0000 max=33138 max_rank=6 min=32427 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32427, 32817, 32705, 32799, 32989, 33138, 32644] avg=32768.0000 max=33138 max_rank=6 min=32427 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[32824, 32544, 32990, 32711, 32917, 32779, 32779, 32600] avg=32768.0000 max=32990 max_rank=2 min=32544 min_rank=1 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32427, 32817, 32705, 32799, 32989, 33138, 32644] avg=32768.0000 max=33138 max_rank=6 min=32427 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32427, 32817, 32705, 32799, 32989, 33138, 32644] avg=32768.0000 max=33138 max_rank=6 min=32427 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32427, 32817, 32705, 32799, 32989, 33138, 32644] avg=32768.0000 max=33138 max_rank=6 min=32427 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32427, 32817, 32705, 32799, 32989, 33138, 32644] avg=32768.0000 max=33138 max_rank=6 min=32427 min_rank=1 imbalance_ratio=1.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32625, 32469, 32817, 32705, 32799, 32989, 33096, 32644] avg=32768.0000 max=33096 max_rank=6 min=32469 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=42 max_load_reduction_pct=0.001267 moved_to_replicas=42 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[32446, 32902, 32747, 33035, 32737, 32656, 32908, 32713] avg=32768.0000 max=33035 max_rank=3 min=32446 min_rank=0 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32914, 33025, 32764, 32855, 32770, 32322, 32811, 32683] avg=32768.0000 max=33025 max_rank=1 min=32322 min_rank=5 imbalance_ratio=1.007843 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=234 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32960, 32700, 32416, 32833, 32900, 32514, 32941, 32880] avg=32768.0000 max=32960 max_rank=0 min=32416 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=234 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=234 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=234 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=234 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=234 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=234 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[32705, 32582, 32636, 32704, 32896, 32923, 32780, 32918] avg=32768.0000 max=32923 max_rank=5 min=32582 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=234 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[32827, 32494, 32784, 32836, 32856, 32509, 32889, 32949] avg=32768.0000 max=32949 max_rank=7 min=32494 min_rank=1 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[32844, 32831, 32677, 32900, 32619, 32532, 32697, 33044] avg=32768.0000 max=33044 max_rank=7 min=32532 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[32753, 32496, 32568, 32868, 32802, 32738, 32957, 32962] avg=32768.0000 max=32962 max_rank=7 min=32496 min_rank=1 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32826, 32916, 32794, 32657, 32902, 32458, 32821, 32770] avg=32768.0000 max=32916 max_rank=1 min=32458 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32745, 32614, 32828, 33034, 32829, 32538, 32900, 32656] avg=32768.0000 max=33034 max_rank=3 min=32538 min_rank=5 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[32645, 32976, 32852, 32790, 32559, 32803, 32705, 32814] avg=32768.0000 max=32976 max_rank=1 min=32559 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[32548, 33048, 32679, 32876, 32805, 32721, 32774, 32693] avg=32768.0000 max=33048 max_rank=1 min=32548 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=1048512 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253824 peak_demand=32996 required_capacity=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=1049022 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253440 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=1047725 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254848 peak_demand=33048 required_capacity=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=1047744 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254976 peak_demand=33044 required_capacity=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=1049281 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253696 peak_demand=33037 required_capacity=1.008209 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=1048380 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253952 peak_demand=32993 required_capacity=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=1046865 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255488 peak_demand=32989 required_capacity=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=1051079 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251776 peak_demand=33035 required_capacity=1.008148 + [2026-08-30 05:12:56] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 423.9 | throughput per GPU (TFLOP/s/GPU): 786.5 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.274399E+00 | loss scale: 1.0 | grad norm: 5.062 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65718 expected_overflow=0 exact_recv_per_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65828 expected_overflow=0 exact_recv_per_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65786 expected_overflow=0 exact_recv_per_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65245 expected_overflow=0 exact_recv_per_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65734 expected_overflow=0 exact_recv_per_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65718 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65286 expected_overflow=0 exact_recv_per_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65828 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65786 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65245 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65734 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65286 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65148 expected_overflow=0 exact_recv_per_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65502 expected_overflow=0 exact_recv_per_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65148 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65349 expected_overflow=0 exact_recv_per_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65616 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65502 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65388 expected_overflow=0 exact_recv_per_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65467 expected_overflow=0 exact_recv_per_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65349 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65653 expected_overflow=0 exact_recv_per_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65388 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65467 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65653 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65510 expected_overflow=0 exact_recv_per_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65263 expected_overflow=0 exact_recv_per_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65788 expected_overflow=0 exact_recv_per_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65510 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65263 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65788 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65773 expected_overflow=0 exact_recv_per_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65766 expected_overflow=0 exact_recv_per_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65676 expected_overflow=0 exact_recv_per_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65577 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65467 expected_overflow=0 exact_recv_per_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65773 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65766 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65676 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65467 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65979 expected_overflow=0 exact_recv_per_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65256 expected_overflow=0 exact_recv_per_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65156 expected_overflow=0 exact_recv_per_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65474 expected_overflow=0 exact_recv_per_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65979 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65256 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65249 expected_overflow=0 exact_recv_per_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65156 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65624 expected_overflow=0 exact_recv_per_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65640 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65474 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65249 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65624 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65790 expected_overflow=0 exact_recv_per_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65608 expected_overflow=0 exact_recv_per_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65492 expected_overflow=0 exact_recv_per_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65790 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65608 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65319 expected_overflow=0 exact_recv_per_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65492 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65448 expected_overflow=0 exact_recv_per_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65659 expected_overflow=0 exact_recv_per_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65319 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65498 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65307 expected_overflow=0 exact_recv_per_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65448 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65659 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65307 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65637 expected_overflow=0 exact_recv_per_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65775 expected_overflow=0 exact_recv_per_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65637 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65775 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65266 expected_overflow=0 exact_recv_per_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65715 expected_overflow=0 exact_recv_per_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65619 expected_overflow=0 exact_recv_per_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65362 expected_overflow=0 exact_recv_per_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65266 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65802 expected_overflow=0 exact_recv_per_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65715 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65619 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65495 expected_overflow=0 exact_recv_per_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65362 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65802 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65495 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65840 expected_overflow=0 exact_recv_per_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65392 expected_overflow=0 exact_recv_per_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65840 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65392 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65897 expected_overflow=0 exact_recv_per_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65603 expected_overflow=0 exact_recv_per_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65350 expected_overflow=0 exact_recv_per_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65897 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65603 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65470 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65350 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65171 expected_overflow=0 exact_recv_per_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65171 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65225 expected_overflow=0 exact_recv_per_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65399 expected_overflow=0 exact_recv_per_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65225 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65399 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65825 expected_overflow=0 exact_recv_per_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65684 expected_overflow=0 exact_recv_per_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65277 expected_overflow=0 exact_recv_per_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65661 expected_overflow=0 exact_recv_per_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65322 expected_overflow=0 exact_recv_per_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65825 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65684 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65277 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65661 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65322 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65647 expected_overflow=0 exact_recv_per_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65647 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65647 expected_overflow=0 exact_recv_per_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65647 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65477 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65563 expected_overflow=0 exact_recv_per_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65286 expected_overflow=0 exact_recv_per_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65348 expected_overflow=0 exact_recv_per_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65654 expected_overflow=0 exact_recv_per_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65642 expected_overflow=0 exact_recv_per_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65563 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65286 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65348 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65654 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65642 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65774 expected_overflow=0 exact_recv_per_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65720 expected_overflow=0 exact_recv_per_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65544 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65774 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65720 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65664 expected_overflow=0 exact_recv_per_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65649 expected_overflow=0 exact_recv_per_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65396 expected_overflow=0 exact_recv_per_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65755 expected_overflow=0 exact_recv_per_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65616 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65664 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65649 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65396 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65755 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65453 expected_overflow=0 exact_recv_per_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65035 expected_overflow=0 exact_recv_per_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65194 expected_overflow=0 exact_recv_per_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65453 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65035 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65194 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65274 expected_overflow=0 exact_recv_per_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65355 expected_overflow=0 exact_recv_per_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65904 expected_overflow=0 exact_recv_per_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65274 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65355 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65523 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65904 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65937 expected_overflow=0 exact_recv_per_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65565 expected_overflow=0 exact_recv_per_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65937 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65551 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65565 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65687 expected_overflow=0 exact_recv_per_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65675 expected_overflow=0 exact_recv_per_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65216 expected_overflow=0 exact_recv_per_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65701 expected_overflow=0 exact_recv_per_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65424 expected_overflow=0 exact_recv_per_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65687 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65675 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65216 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65701 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65424 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65482 expected_overflow=0 exact_recv_per_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65778 expected_overflow=0 exact_recv_per_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65482 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65778 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65552 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65088 expected_overflow=0 exact_recv_per_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65485 expected_overflow=0 exact_recv_per_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65088 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65282 expected_overflow=0 exact_recv_per_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65485 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65991 expected_overflow=0 exact_recv_per_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65282 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65991 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65518 expected_overflow=0 exact_recv_per_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65518 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65621 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65436 expected_overflow=0 exact_recv_per_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65457 expected_overflow=0 exact_recv_per_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65436 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65429 expected_overflow=0 exact_recv_per_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65457 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65239 expected_overflow=0 exact_recv_per_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65644 expected_overflow=0 exact_recv_per_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65429 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65239 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65926 expected_overflow=0 exact_recv_per_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65644 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65323 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65611 expected_overflow=0 exact_recv_per_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65926 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65323 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65611 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65376 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65821 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65376 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65821 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65325 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65700 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65898 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65325 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65700 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65389 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65898 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65519 expected_overflow=0 exact_recv_per_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65456 expected_overflow=0 exact_recv_per_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65389 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65519 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65456 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65282 expected_overflow=0 exact_recv_per_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65510 expected_overflow=0 exact_recv_per_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65282 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65510 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65343 expected_overflow=0 exact_recv_per_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65405 expected_overflow=0 exact_recv_per_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65343 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65740 expected_overflow=0 exact_recv_per_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65405 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65957 expected_overflow=0 exact_recv_per_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65740 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65957 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=3 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=3 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=3 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=3 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=3 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33135, 32599, 32731, 32884] avg=32768.0000 max=33135 max_rank=4 min=32599 min_rank=5 imbalance_ratio=1.011200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=3 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33135, 32599, 32731, 32884] avg=32768.0000 max=33135 max_rank=4 min=32599 min_rank=5 imbalance_ratio=1.011200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=3 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[32646, 32872, 32792, 32855, 32799, 32732, 32964, 32484] avg=32768.0000 max=32964 max_rank=6 min=32484 min_rank=7 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=3 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33135, 32599, 32731, 32884] avg=32768.0000 max=33135 max_rank=4 min=32599 min_rank=5 imbalance_ratio=1.011200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] avg=32768.0000 max=33096 max_rank=4 min=32638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39 max_load_reduction_pct=0.001177 moved_to_replicas=43 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33135, 32599, 32731, 32884] avg=32768.0000 max=33135 max_rank=4 min=32599 min_rank=5 imbalance_ratio=1.011200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33135, 32599, 32731, 32884] avg=32768.0000 max=33135 max_rank=4 min=32599 min_rank=5 imbalance_ratio=1.011200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] avg=32768.0000 max=33096 max_rank=4 min=32638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39 max_load_reduction_pct=0.001177 moved_to_replicas=43 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] avg=32768.0000 max=33096 max_rank=4 min=32638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39 max_load_reduction_pct=0.001177 moved_to_replicas=43 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33135, 32599, 32731, 32884] avg=32768.0000 max=33135 max_rank=4 min=32599 min_rank=5 imbalance_ratio=1.011200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33135, 32599, 32731, 32884] avg=32768.0000 max=33135 max_rank=4 min=32599 min_rank=5 imbalance_ratio=1.011200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] avg=32768.0000 max=33096 max_rank=4 min=32638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39 max_load_reduction_pct=0.001177 moved_to_replicas=43 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33135, 32599, 32731, 32884] avg=32768.0000 max=33135 max_rank=4 min=32599 min_rank=5 imbalance_ratio=1.011200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] avg=32768.0000 max=33096 max_rank=4 min=32638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39 max_load_reduction_pct=0.001177 moved_to_replicas=43 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] avg=32768.0000 max=33096 max_rank=4 min=32638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39 max_load_reduction_pct=0.001177 moved_to_replicas=43 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] avg=32768.0000 max=33096 max_rank=4 min=32638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39 max_load_reduction_pct=0.001177 moved_to_replicas=43 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[32692, 32760, 32651, 32692, 33096, 32638, 32731, 32884] avg=32768.0000 max=33096 max_rank=4 min=32638 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=39 max_load_reduction_pct=0.001177 moved_to_replicas=43 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32635, 32881, 32649, 32835, 32821, 32725, 32801, 32797] avg=32768.0000 max=32881 max_rank=1 min=32635 min_rank=0 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[32712, 32432, 33092, 33017, 32653, 32870, 32537, 32831] avg=32768.0000 max=33092 max_rank=2 min=32432 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32852, 32778, 32978, 32798, 32739, 32807, 32708, 32484] avg=32768.0000 max=32978 max_rank=2 min=32484 min_rank=7 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[32598, 32689, 32798, 32722, 32801, 33001, 32786, 32749] avg=32768.0000 max=33001 max_rank=5 min=32598 min_rank=0 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32577, 32861, 32898, 32676, 32740, 32645, 32713, 33034] avg=32768.0000 max=33034 max_rank=7 min=32577 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32959, 32784, 32716, 32635, 32712, 32613, 32696, 33029] avg=32768.0000 max=33029 max_rank=7 min=32613 min_rank=5 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[32831, 32759, 32727, 32699, 32851, 32645, 32986, 32646] avg=32768.0000 max=32986 max_rank=6 min=32645 min_rank=5 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32573, 32826, 32881, 32710, 32883, 32714, 32879, 32678] avg=32768.0000 max=32883 max_rank=4 min=32573 min_rank=0 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[32685, 32818, 32603, 33075, 32415, 32960, 32858, 32730] avg=32768.0000 max=33075 max_rank=3 min=32415 min_rank=4 imbalance_ratio=1.009369 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32733, 32826, 32714, 32666, 32828, 32799, 32928, 32650] avg=32768.0000 max=32928 max_rank=6 min=32650 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[32839, 32586, 32843, 32938, 32577, 32983, 32626, 32752] avg=32768.0000 max=32983 max_rank=5 min=32577 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32573, 32735, 32650, 32660, 32755, 32950, 32610, 33211] avg=32768.0000 max=33211 max_rank=7 min=32573 min_rank=0 imbalance_ratio=1.013519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32573, 32735, 32650, 32660, 32755, 32950, 32610, 33211] avg=32768.0000 max=33211 max_rank=7 min=32573 min_rank=0 imbalance_ratio=1.013519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32573, 32735, 32650, 32660, 32755, 32950, 32610, 33211] avg=32768.0000 max=33211 max_rank=7 min=32573 min_rank=0 imbalance_ratio=1.013519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32573, 32735, 32650, 32660, 32755, 32950, 32610, 33211] avg=32768.0000 max=33211 max_rank=7 min=32573 min_rank=0 imbalance_ratio=1.013519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32573, 32735, 32650, 32660, 32755, 32950, 32610, 33211] avg=32768.0000 max=33211 max_rank=7 min=32573 min_rank=0 imbalance_ratio=1.013519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[32894, 32605, 32905, 32893, 32744, 32785, 32603, 32715] avg=32768.0000 max=32905 max_rank=2 min=32603 min_rank=6 imbalance_ratio=1.004181 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] avg=32768.0000 max=33096 max_rank=7 min=32610 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=115 max_load_reduction_pct=0.003463 moved_to_replicas=115 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] avg=32768.0000 max=33096 max_rank=7 min=32610 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=115 max_load_reduction_pct=0.003463 moved_to_replicas=115 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] avg=32768.0000 max=33096 max_rank=7 min=32610 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=115 max_load_reduction_pct=0.003463 moved_to_replicas=115 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] avg=32768.0000 max=33096 max_rank=7 min=32610 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=115 max_load_reduction_pct=0.003463 moved_to_replicas=115 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] avg=32768.0000 max=33096 max_rank=7 min=32610 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=115 max_load_reduction_pct=0.003463 moved_to_replicas=115 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32573, 32735, 32650, 32660, 32755, 32950, 32610, 33211] avg=32768.0000 max=33211 max_rank=7 min=32573 min_rank=0 imbalance_ratio=1.013519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32573, 32735, 32650, 32660, 32755, 32950, 32610, 33211] avg=32768.0000 max=33211 max_rank=7 min=32573 min_rank=0 imbalance_ratio=1.013519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32573, 32735, 32650, 32660, 32755, 32950, 32610, 33211] avg=32768.0000 max=33211 max_rank=7 min=32573 min_rank=0 imbalance_ratio=1.013519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] avg=32768.0000 max=33096 max_rank=7 min=32610 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=115 max_load_reduction_pct=0.003463 moved_to_replicas=115 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] avg=32768.0000 max=33096 max_rank=7 min=32610 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=115 max_load_reduction_pct=0.003463 moved_to_replicas=115 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32688, 32735, 32650, 32660, 32755, 32950, 32610, 33096] avg=32768.0000 max=33096 max_rank=7 min=32610 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=115 max_load_reduction_pct=0.003463 moved_to_replicas=115 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[32861, 33017, 32595, 32799, 32583, 32771, 32668, 32850] avg=32768.0000 max=33017 max_rank=1 min=32583 min_rank=4 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=1047573 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254592 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=1048887 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253696 peak_demand=33092 required_capacity=1.009888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=1048252 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255232 peak_demand=32986 required_capacity=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=1048115 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254720 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=1048623 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253568 peak_demand=33086 required_capacity=1.009705 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=1049401 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253184 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=1049173 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253184 peak_demand=33075 required_capacity=1.009369 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=1048584 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254080 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:12:57] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 430.6 | throughput per GPU (TFLOP/s/GPU): 774.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.133406E+00 | loss scale: 1.0 | grad norm: 4.530 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65276 expected_overflow=0 exact_recv_per_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65460 expected_overflow=0 exact_recv_per_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65459 expected_overflow=0 exact_recv_per_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65611 expected_overflow=0 exact_recv_per_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65165 expected_overflow=0 exact_recv_per_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65276 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65935 expected_overflow=0 exact_recv_per_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65460 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65459 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65829 expected_overflow=0 exact_recv_per_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65611 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65553 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65165 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65935 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65829 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=40960 expected_recv=66052 expected_overflow=0 exact_recv_per_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65645 expected_overflow=0 exact_recv_per_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65199 expected_overflow=0 exact_recv_per_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=40960 expected_recv=66052 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65516 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65645 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65729 expected_overflow=0 exact_recv_per_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65199 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65509 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65729 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65249 expected_overflow=0 exact_recv_per_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65389 expected_overflow=0 exact_recv_per_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65249 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65389 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65658 expected_overflow=0 exact_recv_per_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65589 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65529 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65745 expected_overflow=0 exact_recv_per_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65710 expected_overflow=0 exact_recv_per_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65658 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65508 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65745 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65710 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65219 expected_overflow=0 exact_recv_per_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65330 expected_overflow=0 exact_recv_per_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65219 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65330 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65162 expected_overflow=0 exact_recv_per_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65281 expected_overflow=0 exact_recv_per_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65162 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65281 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65596 expected_overflow=0 exact_recv_per_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65613 expected_overflow=0 exact_recv_per_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65643 expected_overflow=0 exact_recv_per_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65651 expected_overflow=0 exact_recv_per_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65596 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65613 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65643 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65651 expected_overflow=0 exact_recv_per_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65691 expected_overflow=0 exact_recv_per_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65651 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65651 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65691 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65927 expected_overflow=0 exact_recv_per_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65472 expected_overflow=0 exact_recv_per_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65927 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65472 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65291 expected_overflow=0 exact_recv_per_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65495 expected_overflow=0 exact_recv_per_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65678 expected_overflow=0 exact_recv_per_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65557 expected_overflow=0 exact_recv_per_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65283 expected_overflow=0 exact_recv_per_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65585 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65291 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65495 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65678 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65557 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65283 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65984 expected_overflow=0 exact_recv_per_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65387 expected_overflow=0 exact_recv_per_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65984 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65008 expected_overflow=0 exact_recv_per_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65387 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65688 expected_overflow=0 exact_recv_per_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65612 expected_overflow=0 exact_recv_per_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65346 expected_overflow=0 exact_recv_per_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65008 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65734 expected_overflow=0 exact_recv_per_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65688 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65612 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65346 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65734 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65529 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65703 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65338 expected_overflow=0 exact_recv_per_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65429 expected_overflow=0 exact_recv_per_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65697 expected_overflow=0 exact_recv_per_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65338 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65643 expected_overflow=0 exact_recv_per_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65429 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65697 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65643 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65407 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65666 expected_overflow=0 exact_recv_per_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65666 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65553 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65634 expected_overflow=0 exact_recv_per_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65741 expected_overflow=0 exact_recv_per_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65217 expected_overflow=0 exact_recv_per_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65634 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65446 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65741 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65217 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65528 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65503 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65455 expected_overflow=0 exact_recv_per_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65590 expected_overflow=0 exact_recv_per_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65455 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65590 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65381 expected_overflow=0 exact_recv_per_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65298 expected_overflow=0 exact_recv_per_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65381 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=40960 expected_recv=66021 expected_overflow=0 exact_recv_per_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65161 expected_overflow=0 exact_recv_per_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65298 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65835 expected_overflow=0 exact_recv_per_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65547 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=40960 expected_recv=66021 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65161 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65455 expected_overflow=0 exact_recv_per_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65906 expected_overflow=0 exact_recv_per_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65835 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65455 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65906 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65374 expected_overflow=0 exact_recv_per_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65374 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65586 expected_overflow=0 exact_recv_per_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65286 expected_overflow=0 exact_recv_per_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65586 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65386 expected_overflow=0 exact_recv_per_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65687 expected_overflow=0 exact_recv_per_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65286 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65608 expected_overflow=0 exact_recv_per_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65623 expected_overflow=0 exact_recv_per_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65386 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65687 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65608 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65623 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65531 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65932 expected_overflow=0 exact_recv_per_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65932 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65239 expected_overflow=0 exact_recv_per_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65340 expected_overflow=0 exact_recv_per_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65239 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65599 expected_overflow=0 exact_recv_per_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65340 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65563 expected_overflow=0 exact_recv_per_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65461 expected_overflow=0 exact_recv_per_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65153 expected_overflow=0 exact_recv_per_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65619 expected_overflow=0 exact_recv_per_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65599 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65563 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65461 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65153 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65619 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65684 expected_overflow=0 exact_recv_per_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65684 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65739 expected_overflow=0 exact_recv_per_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65402 expected_overflow=0 exact_recv_per_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65739 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65402 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65257 expected_overflow=0 exact_recv_per_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65636 expected_overflow=0 exact_recv_per_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65470 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65585 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65257 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65741 expected_overflow=0 exact_recv_per_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65636 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65563 expected_overflow=0 exact_recv_per_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65741 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65563 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65444 expected_overflow=0 exact_recv_per_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65544 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65951 expected_overflow=0 exact_recv_per_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65444 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65400 expected_overflow=0 exact_recv_per_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65380 expected_overflow=0 exact_recv_per_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65408 expected_overflow=0 exact_recv_per_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65951 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65400 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65226 expected_overflow=0 exact_recv_per_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65380 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65408 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65754 expected_overflow=0 exact_recv_per_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65226 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65754 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65480 expected_overflow=0 exact_recv_per_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65116 expected_overflow=0 exact_recv_per_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65480 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65116 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65742 expected_overflow=0 exact_recv_per_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65347 expected_overflow=0 exact_recv_per_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65742 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65913 expected_overflow=0 exact_recv_per_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65657 expected_overflow=0 exact_recv_per_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65873 expected_overflow=0 exact_recv_per_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65383 expected_overflow=0 exact_recv_per_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65347 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65913 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65657 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65873 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65383 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65330 expected_overflow=0 exact_recv_per_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65498 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65330 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65782 expected_overflow=0 exact_recv_per_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65420 expected_overflow=0 exact_recv_per_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65782 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65622 expected_overflow=0 exact_recv_per_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65420 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65520 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65517 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65622 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65763 expected_overflow=0 exact_recv_per_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65763 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65539 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65497 expected_overflow=0 exact_recv_per_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65497 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65200 expected_overflow=0 exact_recv_per_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65727 expected_overflow=0 exact_recv_per_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65200 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65727 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=48 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=48 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[32662, 32772, 32658, 32660, 33021, 32967, 32609, 32795] avg=32768.0000 max=33021 max_rank=4 min=32609 min_rank=6 imbalance_ratio=1.007721 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=48 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=48 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=48 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=48 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=48 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[32956, 32810, 32835, 32531, 32575, 32669, 32926, 32842] avg=32768.0000 max=32956 max_rank=0 min=32531 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=48 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32756, 32818, 32829, 32810, 32607, 32679, 32731, 32914] avg=32768.0000 max=32914 max_rank=7 min=32607 min_rank=4 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=83 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=83 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=83 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=83 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=83 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[32649, 32774, 32723, 32763, 32844, 32845, 32787, 32759] avg=32768.0000 max=32845 max_rank=5 min=32649 min_rank=0 imbalance_ratio=1.002350 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=83 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=83 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[32831, 32825, 32657, 32676, 32668, 32763, 32775, 32949] avg=32768.0000 max=32949 max_rank=7 min=32657 min_rank=2 imbalance_ratio=1.005524 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=83 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[32997, 32698, 32924, 32768, 32951, 32909, 32558, 32339] avg=32768.0000 max=32997 max_rank=0 min=32339 min_rank=7 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32673, 32889, 32680, 32838, 32912, 32581, 32924, 32647] avg=32768.0000 max=32924 max_rank=6 min=32581 min_rank=5 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[32903, 32943, 32806, 32962, 32715, 32407, 32620, 32788] avg=32768.0000 max=32962 max_rank=3 min=32407 min_rank=5 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=27 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32750, 32783, 32878, 33040, 32496, 32744, 32765, 32688] avg=32768.0000 max=33040 max_rank=3 min=32496 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32862, 32662, 32746, 32814, 32625, 32978, 32673, 32784] avg=32768.0000 max=32978 max_rank=5 min=32625 min_rank=4 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[32732, 32854, 32545, 32900, 32827, 32616, 32703, 32967] avg=32768.0000 max=32967 max_rank=7 min=32545 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[32758, 32702, 32989, 32587, 32708, 32638, 32809, 32953] avg=32768.0000 max=32989 max_rank=2 min=32587 min_rank=3 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=72 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=72 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=72 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=72 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=72 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=72 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[32875, 32462, 32966, 32641, 32937, 32639, 32895, 32729] avg=32768.0000 max=32966 max_rank=2 min=32462 min_rank=1 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=72 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[32675, 32614, 32879, 32775, 32646, 33015, 32485, 33055] avg=32768.0000 max=33055 max_rank=7 min=32485 min_rank=6 imbalance_ratio=1.008759 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=72 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[32927, 32655, 32737, 32710, 32716, 32767, 32695, 32937] avg=32768.0000 max=32937 max_rank=7 min=32655 min_rank=1 imbalance_ratio=1.005157 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[32766, 32707, 32803, 32882, 32715, 32561, 32869, 32841] avg=32768.0000 max=32882 max_rank=3 min=32561 min_rank=5 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=1048708 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253312 peak_demand=33055 required_capacity=1.008759 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=1047146 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255616 peak_demand=32943 required_capacity=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=1047197 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255488 peak_demand=32979 required_capacity=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=1048359 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254080 peak_demand=32989 required_capacity=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=1050073 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252288 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=1048646 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254336 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=1049559 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252928 peak_demand=33040 required_capacity=1.008301 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=1048920 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253056 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:12:57] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 430.4 | throughput per GPU (TFLOP/s/GPU): 774.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.041297E+00 | loss scale: 1.0 | grad norm: 4.487 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65657 expected_overflow=0 exact_recv_per_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65666 expected_overflow=0 exact_recv_per_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65487 expected_overflow=0 exact_recv_per_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65266 expected_overflow=0 exact_recv_per_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65417 expected_overflow=0 exact_recv_per_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65657 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65666 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65593 expected_overflow=0 exact_recv_per_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65487 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65266 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65385 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65417 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65593 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65817 expected_overflow=0 exact_recv_per_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65846 expected_overflow=0 exact_recv_per_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65745 expected_overflow=0 exact_recv_per_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65811 expected_overflow=0 exact_recv_per_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65380 expected_overflow=0 exact_recv_per_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65846 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65745 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65811 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65331 expected_overflow=0 exact_recv_per_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65077 expected_overflow=0 exact_recv_per_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65380 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65817 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65415 expected_overflow=0 exact_recv_per_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65331 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65077 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65415 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65387 expected_overflow=0 exact_recv_per_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65158 expected_overflow=0 exact_recv_per_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65926 expected_overflow=0 exact_recv_per_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65387 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65158 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65926 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65733 expected_overflow=0 exact_recv_per_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65683 expected_overflow=0 exact_recv_per_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65328 expected_overflow=0 exact_recv_per_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65446 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65733 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65683 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65328 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65118 expected_overflow=0 exact_recv_per_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65936 expected_overflow=0 exact_recv_per_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65352 expected_overflow=0 exact_recv_per_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65648 expected_overflow=0 exact_recv_per_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65118 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65936 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65352 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65762 expected_overflow=0 exact_recv_per_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65648 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65469 expected_overflow=0 exact_recv_per_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65777 expected_overflow=0 exact_recv_per_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65762 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65469 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65580 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65777 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65670 expected_overflow=0 exact_recv_per_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65261 expected_overflow=0 exact_recv_per_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65506 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65670 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65407 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65261 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65459 expected_overflow=0 exact_recv_per_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65696 expected_overflow=0 exact_recv_per_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65423 expected_overflow=0 exact_recv_per_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65459 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65696 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65598 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65423 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65687 expected_overflow=0 exact_recv_per_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65228 expected_overflow=0 exact_recv_per_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65596 expected_overflow=0 exact_recv_per_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65655 expected_overflow=0 exact_recv_per_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65687 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65228 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65596 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65655 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65382 expected_overflow=0 exact_recv_per_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65382 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65691 expected_overflow=0 exact_recv_per_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65511 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65691 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65616 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65721 expected_overflow=0 exact_recv_per_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65639 expected_overflow=0 exact_recv_per_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65053 expected_overflow=0 exact_recv_per_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65497 expected_overflow=0 exact_recv_per_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65721 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65639 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65053 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65497 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65369 expected_overflow=0 exact_recv_per_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65461 expected_overflow=0 exact_recv_per_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65369 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65613 expected_overflow=0 exact_recv_per_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65461 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65777 expected_overflow=0 exact_recv_per_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65613 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65777 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65784 expected_overflow=0 exact_recv_per_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65820 expected_overflow=0 exact_recv_per_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65463 expected_overflow=0 exact_recv_per_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65316 expected_overflow=0 exact_recv_per_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65784 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65820 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65463 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65480 expected_overflow=0 exact_recv_per_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65316 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65690 expected_overflow=0 exact_recv_per_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65480 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65771 expected_overflow=0 exact_recv_per_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65507 expected_overflow=0 exact_recv_per_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65690 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65771 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65507 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65780 expected_overflow=0 exact_recv_per_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65355 expected_overflow=0 exact_recv_per_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65733 expected_overflow=0 exact_recv_per_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65780 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65355 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65733 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65621 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65388 expected_overflow=0 exact_recv_per_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65388 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65228 expected_overflow=0 exact_recv_per_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65710 expected_overflow=0 exact_recv_per_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65300 expected_overflow=0 exact_recv_per_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=40960 expected_recv=65228 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65710 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65300 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65619 expected_overflow=0 exact_recv_per_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65298 expected_overflow=0 exact_recv_per_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65454 expected_overflow=0 exact_recv_per_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65564 expected_overflow=0 exact_recv_per_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65619 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65298 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65454 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65564 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65484 expected_overflow=0 exact_recv_per_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65484 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65401 expected_overflow=0 exact_recv_per_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65449 expected_overflow=0 exact_recv_per_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65903 expected_overflow=0 exact_recv_per_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65401 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65449 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65903 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65713 expected_overflow=0 exact_recv_per_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65315 expected_overflow=0 exact_recv_per_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65600 expected_overflow=0 exact_recv_per_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65713 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65315 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65600 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65529 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65257 expected_overflow=0 exact_recv_per_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65602 expected_overflow=0 exact_recv_per_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65257 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65517 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65602 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65716 expected_overflow=0 exact_recv_per_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65597 expected_overflow=0 exact_recv_per_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65427 expected_overflow=0 exact_recv_per_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65566 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65716 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65597 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65427 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65722 expected_overflow=0 exact_recv_per_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65609 expected_overflow=0 exact_recv_per_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65794 expected_overflow=0 exact_recv_per_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65722 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65609 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65794 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65293 expected_overflow=0 exact_recv_per_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65483 expected_overflow=0 exact_recv_per_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65242 expected_overflow=0 exact_recv_per_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65293 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65483 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65548 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65521 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65242 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65286 expected_overflow=0 exact_recv_per_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65683 expected_overflow=0 exact_recv_per_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65751 expected_overflow=0 exact_recv_per_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65286 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65683 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65751 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65388 expected_overflow=0 exact_recv_per_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65363 expected_overflow=0 exact_recv_per_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65211 expected_overflow=0 exact_recv_per_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65647 expected_overflow=0 exact_recv_per_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65743 expected_overflow=0 exact_recv_per_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65388 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65363 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65211 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65647 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65743 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65697 expected_overflow=0 exact_recv_per_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65475 expected_overflow=0 exact_recv_per_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65697 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65475 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65501 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65153 expected_overflow=0 exact_recv_per_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65641 expected_overflow=0 exact_recv_per_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=40960 expected_recv=66028 expected_overflow=0 exact_recv_per_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65293 expected_overflow=0 exact_recv_per_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65153 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65641 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=40960 expected_recv=66028 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65293 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65295 expected_overflow=0 exact_recv_per_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65882 expected_overflow=0 exact_recv_per_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65667 expected_overflow=0 exact_recv_per_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65295 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65882 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65667 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65803 expected_overflow=0 exact_recv_per_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65402 expected_overflow=0 exact_recv_per_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65803 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65469 expected_overflow=0 exact_recv_per_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65797 expected_overflow=0 exact_recv_per_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65527 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65402 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65469 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65316 expected_overflow=0 exact_recv_per_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65797 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65396 expected_overflow=0 exact_recv_per_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65316 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65396 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65534 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65830 expected_overflow=0 exact_recv_per_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65327 expected_overflow=0 exact_recv_per_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65830 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65327 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32640, 32677, 32895, 32801, 32706, 32424, 32918, 33083] avg=32768.0000 max=33083 max_rank=7 min=32424 min_rank=5 imbalance_ratio=1.009613 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32872, 32936, 33052, 32678, 32745, 32467, 32628, 32766] avg=32768.0000 max=33052 max_rank=2 min=32467 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32623, 32596, 32807, 32850, 32698, 32848, 33023, 32699] avg=32768.0000 max=33023 max_rank=6 min=32596 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32317, 32606, 33077, 32897, 32815, 32971, 32806, 32655] avg=32768.0000 max=33077 max_rank=2 min=32317 min_rank=0 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[32914, 32668, 32789, 32815, 32750, 32732, 32598, 32878] avg=32768.0000 max=32914 max_rank=0 min=32598 min_rank=6 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[32553, 32770, 32844, 32804, 32677, 32850, 32899, 32747] avg=32768.0000 max=32899 max_rank=6 min=32553 min_rank=0 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32597, 32836, 32703, 32903, 32562, 32837, 32808, 32898] avg=32768.0000 max=32903 max_rank=3 min=32562 min_rank=4 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32898, 32851, 32756, 32700, 33001, 32666, 32703, 32569] avg=32768.0000 max=33001 max_rank=4 min=32569 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32631, 32980, 33007, 32647, 32683, 32664, 32895, 32637] avg=32768.0000 max=33007 max_rank=2 min=32631 min_rank=0 imbalance_ratio=1.007294 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[32720, 32823, 32812, 33049, 32807, 32682, 32536, 32715] avg=32768.0000 max=33049 max_rank=3 min=32536 min_rank=6 imbalance_ratio=1.008575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[32507, 32827, 32921, 32767, 32610, 32783, 32659, 33070] avg=32768.0000 max=33070 max_rank=7 min=32507 min_rank=0 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32810, 32757, 32804, 32629, 32753, 32743, 32881, 32767] avg=32768.0000 max=32881 max_rank=6 min=32629 min_rank=3 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32507, 32974, 32833, 32698, 32763, 32863, 32630, 32876] avg=32768.0000 max=32974 max_rank=1 min=32507 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[32854, 32962, 32461, 32953, 32714, 32800, 32586, 32814] avg=32768.0000 max=32962 max_rank=1 min=32461 min_rank=2 imbalance_ratio=1.005920 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32737, 32782, 32412, 32939, 32897, 32731, 32770, 32876] avg=32768.0000 max=32939 max_rank=3 min=32412 min_rank=2 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[32620, 32710, 33011, 32831, 32746, 32872, 32649, 32705] avg=32768.0000 max=33011 max_rank=2 min=32620 min_rank=0 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=1048047 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254336 peak_demand=33023 required_capacity=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=1049937 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252800 peak_demand=33077 required_capacity=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=1049080 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253568 peak_demand=33027 required_capacity=1.007904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=1048609 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254848 peak_demand=32980 required_capacity=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=1047990 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254720 peak_demand=33051 required_capacity=1.008636 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=1047123 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255872 peak_demand=33022 required_capacity=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=1048564 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253696 peak_demand=33083 required_capacity=1.009613 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=1049258 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252928 peak_demand=33075 required_capacity=1.009369 + [2026-08-30 05:12:58] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 423.5 | throughput per GPU (TFLOP/s/GPU): 787.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.947520E+00 | loss scale: 1.0 | grad norm: 4.897 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +[after training is done] datetime: 2026-08-30 05:12:58 +[rank0]:[W830 05:12:59.545740857 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 05:13:00.074553067 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 05:13:00.077218816 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 05:13:00.077384687 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 05:13:00.077953665 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 05:13:00.191599982 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 05:13:01.493288581 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 05:13:01.495367563 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:13:07.508000 492296 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:13:07.509000 492296 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 492393 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 492296 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..cd23c0d805152d51ada9f4f145691006e6c89748 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 27 +2, 27 +3, 4 +4, 4 +5, 4 +6, 4 +7, 45 +0, 887 +1, 895 +2, 879 +3, 889 +4, 911 +5, 883 +6, 887 +7, 889 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 2333 +1, 2337 +2, 2335 +3, 2343 +4, 2343 +5, 2373 +6, 2361 +7, 2369 +0, 12791 +1, 14867 +2, 12787 +3, 12791 +4, 12791 +5, 12783 +6, 35145 +7, 19017 +0, 42215 +1, 42203 +2, 42203 +3, 42203 +4, 42203 +5, 42203 +6, 42211 +7, 42211 +0, 42941 +1, 42915 +2, 42935 +3, 42925 +4, 42935 +5, 42945 +6, 42945 +7, 42945 +0, 43545 +1, 43529 +2, 43529 +3, 43529 +4, 43529 +5, 43529 +6, 43529 +7, 43529 +0, 44089 +1, 44073 +2, 44073 +3, 44073 +4, 44073 +5, 44073 +6, 44073 +7, 44073 +0, 45919 +1, 45903 +2, 45903 +3, 45903 +4, 45903 +5, 45903 +6, 45903 +7, 45903 +0, 45919 +1, 45903 +2, 45903 +3, 45903 +4, 45903 +5, 45903 +6, 45903 +7, 45903 +0, 73899 +1, 73809 +2, 73809 +3, 73809 +4, 73859 +5, 73809 +6, 73809 +7, 73809 +0, 73899 +1, 73809 +2, 73809 +3, 73809 +4, 73859 +5, 73809 +6, 73809 +7, 73809 +0, 73899 +1, 73811 +2, 73811 +3, 73811 +4, 73859 +5, 73811 +6, 73811 +7, 73811 +0, 73913 +1, 73825 +2, 73825 +3, 73825 +4, 73873 +5, 73825 +6, 73825 +7, 73825 +0, 87097 +1, 87177 +2, 87209 +3, 87253 +4, 87293 +5, 87245 +6, 87243 +7, 87255 +0, 88161 +1, 88235 +2, 88255 +3, 88229 +4, 88263 +5, 88235 +6, 88255 +7, 88259 +0, 106235 +1, 106493 +2, 106333 +3, 106493 +4, 105533 +5, 106333 +6, 106553 +7, 107097 +0, 111521 +1, 111567 +2, 111567 +3, 111567 +4, 111565 +5, 111567 +6, 111567 +7, 111567 +0, 111521 +1, 111567 +2, 111567 +3, 111567 +4, 111565 +5, 111567 +6, 111567 +7, 111567 +0, 111521 +1, 111567 +2, 111567 +3, 111567 +4, 111565 +5, 111567 +6, 111567 +7, 111567 +0, 107381 +1, 107427 +2, 107427 +3, 107427 +4, 107425 +5, 107427 +6, 107501 +7, 107427 +0, 105697 +1, 106393 +2, 105995 +3, 106005 +4, 105741 +5, 105685 +6, 106005 +7, 106393 +0, 1 +1, 4 +2, 132 +3, 106005 +4, 644 +5, 4 +6, 4 +7, 4 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..024db178dd012ab5ddfa720a468f735dc26d6348 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G3 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=1792 +num_experts=64 +topk=16 +experts_per_rank=8 +expert_params=22020096 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=65536 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.0625 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b66fab4757a66c724607658bc76cf08e69cbac18 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/train.log @@ -0,0 +1,9092 @@ +[05:14:24] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G3 / ultraep (slots/rank=8 policy=matched) + E=64 K=16 H'=1792 m=8 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.0625 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:14:25.814000 498940 torch/distributed/run.py:766] +W0830 05:14:25.814000 498940 torch/distributed/run.py:766] ***************************************** +W0830 05:14:25.814000 498940 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:14:25.814000 498940 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1792 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0625 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 16 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6[Gloo] Rank is connected to 072 is connected to peer ranks. is connected to 7Expected number of connected peer ranks is : 7 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : +Expected number of connected peer ranks is : 77 + +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 65 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W830 05:14:43.204216719 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:14:43.204456963 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:14:43.211969794 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:14:43.212037527 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:14:43.212050926 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:14:43.212076815 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:14:43.212087149 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:14:44.367922688 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.882 seconds +time to initialize megatron (seconds): 7.108 +[after megatron is initialized] datetime: 2026-08-30 05:14:49 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6724653056 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42217472 elements, 42217472 padded size): + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (42213376 elements, 42213376 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight +Params for bucket 4 (42213376 elements, 42213376 padded size): + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.pre_mlp_layernorm.weight +Params for bucket 5 (42213376 elements, 42213376 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.pre_mlp_layernorm.weight +Params for bucket 6 (42213376 elements, 42213376 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight +Params for bucket 7 (42213376 elements, 42213376 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 8 (42213376 elements, 42213376 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight +Params for bucket 9 (42213376 elements, 42213376 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 10 (42213376 elements, 42213376 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 11 (42213376 elements, 42213376 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 12 (42213376 elements, 42213376 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight +Params for bucket 13 (42213376 elements, 42213376 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (42213376 elements, 42213376 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (42213376 elements, 42213376 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight +Params for bucket 16 (42213376 elements, 42213376 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 17 (42213376 elements, 42213376 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 64 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 +Params for bucket 17 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 35 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 +Params for bucket 42 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 55 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 +Params for bucket 56 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 63 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:14:53 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:14:53 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3586.51, 3591.46) + train/valid/test-data-iterators-setup ..........: (76.00, 90.20) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:14:53 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=69632 expected_recv=131115 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=69632 expected_recv=131115 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=69632 expected_recv=126378 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=69632 expected_recv=131190 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=69632 expected_recv=130356 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=69632 expected_recv=126378 actual_valid_recv=127360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=69632 expected_recv=131190 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=127360 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=69632 expected_recv=130356 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=69632 expected_recv=129879 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=69632 expected_recv=127091 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=69632 expected_recv=129879 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=69632 expected_recv=129688 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=69632 expected_recv=127091 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=69632 expected_recv=129688 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=128384 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=69632 expected_recv=131069 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=69632 expected_recv=128043 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=69632 expected_recv=131069 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=69632 expected_recv=128043 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=69632 expected_recv=131332 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=129280 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=69632 expected_recv=131332 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=69632 expected_recv=130791 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=69632 expected_recv=130791 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=69632 expected_recv=130137 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=69632 expected_recv=130137 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=69632 expected_recv=130737 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=69632 expected_recv=130737 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=69632 expected_recv=129984 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=69632 expected_recv=130165 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=69632 expected_recv=129984 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=69632 expected_recv=128694 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=69632 expected_recv=130165 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=69632 expected_recv=128694 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=69632 expected_recv=127113 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=69632 expected_recv=127113 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=69632 expected_recv=129831 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=128384 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=69632 expected_recv=126976 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=69632 expected_recv=129831 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=69632 expected_recv=126976 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=69632 expected_recv=129848 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=128128 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=69632 expected_recv=129848 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=69632 expected_recv=128989 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=69632 expected_recv=128101 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=69632 expected_recv=128989 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=69632 expected_recv=128101 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=69632 expected_recv=129418 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=129408 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=69632 expected_recv=129418 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=69632 expected_recv=125849 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=69632 expected_recv=125849 actual_valid_recv=126976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=126976 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=69632 expected_recv=128348 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=69632 expected_recv=130208 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=69632 expected_recv=128348 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=69632 expected_recv=130208 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=129664 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=69632 expected_recv=128476 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=69632 expected_recv=127463 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=69632 expected_recv=128476 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=129408 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=69632 expected_recv=127463 actual_valid_recv=128896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=128896 slack_slots=10368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=69632 expected_recv=131844 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=69632 expected_recv=126494 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=69632 expected_recv=127777 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=69632 expected_recv=126494 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=69632 expected_recv=131844 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=69632 expected_recv=127777 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=128128 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=128768 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=69632 expected_recv=131602 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=69632 expected_recv=129600 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=69632 expected_recv=131602 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=69632 expected_recv=129600 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=69632 expected_recv=132095 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=69632 expected_recv=127192 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=69632 expected_recv=132095 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=69632 expected_recv=131246 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=69632 expected_recv=131519 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=69632 expected_recv=127192 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=69632 expected_recv=131246 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=69632 expected_recv=131519 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=128512 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=69632 expected_recv=128342 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=69632 expected_recv=128456 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=69632 expected_recv=128342 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=69632 expected_recv=128456 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=129792 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=69632 expected_recv=130554 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=69632 expected_recv=131312 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=69632 expected_recv=130554 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=69632 expected_recv=131312 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=69632 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=69632 expected_recv=132381 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=69632 expected_recv=128261 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=69632 expected_recv=128261 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=69632 expected_recv=128426 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=129408 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=69632 expected_recv=128426 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=129792 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=69632 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=69632 expected_recv=132381 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=69632 expected_recv=128910 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=69632 expected_recv=128910 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=69632 expected_recv=130066 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=69632 expected_recv=131262 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=69632 expected_recv=129776 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=69632 expected_recv=130577 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=69632 expected_recv=130066 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=69632 expected_recv=131262 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=69632 expected_recv=129776 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=69632 expected_recv=130577 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=69632 expected_recv=131260 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=69632 expected_recv=130270 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=69632 expected_recv=131260 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=69632 expected_recv=130270 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=69632 expected_recv=130606 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=69632 expected_recv=128409 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=69632 expected_recv=131111 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=69632 expected_recv=130606 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=69632 expected_recv=128409 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=69632 expected_recv=131111 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=129536 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=69632 expected_recv=131192 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=69632 expected_recv=127848 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=69632 expected_recv=131018 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=69632 expected_recv=131192 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=69632 expected_recv=131605 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=69632 expected_recv=127848 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=69632 expected_recv=131018 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=129280 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=69632 expected_recv=131605 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=69632 expected_recv=128812 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=69632 expected_recv=128812 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=69632 expected_recv=129890 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=69632 expected_recv=129890 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=69632 expected_recv=130509 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=69632 expected_recv=130509 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=2101251 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=107776 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=2089759 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=118656 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=2085444 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=122880 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=2095336 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=113024 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=2107131 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=102400 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=2102385 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=108032 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=2102492 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=106112 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=2093418 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=118144 peak_demand=66193 required_capacity=1.010025 + [2026-08-30 05:15:22] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 29118.0 | throughput per GPU (TFLOP/s/GPU): 11.4 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102352E+01 | loss scale: 1.0 | grad norm: 44.192 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67072 | min safe capacity: 1.0234 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58093.3076171875 | max allocated: 76810.0107421875 | reserved: 79350.0 | max reserved: 79350.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=69632 expected_recv=131215 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=69632 expected_recv=129985 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=69632 expected_recv=128740 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=69632 expected_recv=129100 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=69632 expected_recv=131215 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=69632 expected_recv=129985 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=69632 expected_recv=128740 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=69632 expected_recv=129100 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=130048 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=69632 expected_recv=131700 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=69632 expected_recv=131700 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=69632 expected_recv=127240 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=69632 expected_recv=127719 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=69632 expected_recv=127240 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=69632 expected_recv=127719 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=128768 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=129152 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=69632 expected_recv=127528 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=69632 expected_recv=127528 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=129024 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=69632 expected_recv=126744 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=69632 expected_recv=126744 actual_valid_recv=128000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=128000 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=69632 expected_recv=128228 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=69632 expected_recv=128228 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=129664 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=69632 expected_recv=128168 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=69632 expected_recv=128168 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=69632 expected_recv=131353 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=69632 expected_recv=131287 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=129664 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=69632 expected_recv=131287 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=69632 expected_recv=131353 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=69632 expected_recv=129757 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=69632 expected_recv=124536 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132363 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=69632 expected_recv=129757 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=69632 expected_recv=124536 actual_valid_recv=125696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132363 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=125696 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=69632 expected_recv=130421 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=69632 expected_recv=130421 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=69632 expected_recv=130813 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=69632 expected_recv=127711 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=69632 expected_recv=130813 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=69632 expected_recv=130094 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=69632 expected_recv=127711 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=69632 expected_recv=130094 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=129408 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=69632 expected_recv=124774 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=69632 expected_recv=124774 actual_valid_recv=126208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=126208 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=69632 expected_recv=129496 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=69632 expected_recv=128679 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=69632 expected_recv=129496 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=69632 expected_recv=130844 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=69632 expected_recv=128679 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=69632 expected_recv=130844 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=69632 expected_recv=130261 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=69632 expected_recv=130261 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=69632 expected_recv=129259 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=69632 expected_recv=129259 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=69632 expected_recv=130072 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=69632 expected_recv=129448 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=69632 expected_recv=130072 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=69632 expected_recv=129448 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=69632 expected_recv=129804 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=69632 expected_recv=129804 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=69632 expected_recv=130613 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=69632 expected_recv=131488 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=69632 expected_recv=129640 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=69632 expected_recv=130613 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=69632 expected_recv=131488 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=69632 expected_recv=129640 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=69632 expected_recv=131350 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=69632 expected_recv=131350 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=69632 expected_recv=130756 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=69632 expected_recv=132248 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=69632 expected_recv=130756 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=69632 expected_recv=127293 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=69632 expected_recv=128757 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=69632 expected_recv=132248 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=69632 expected_recv=127293 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=69632 expected_recv=128757 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=128512 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=69632 expected_recv=126770 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=69632 expected_recv=126770 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=128128 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=69632 expected_recv=131266 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=69632 expected_recv=131266 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=69632 expected_recv=129913 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=69632 expected_recv=129913 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=69632 expected_recv=125982 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=69632 expected_recv=125982 actual_valid_recv=127360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=69632 expected_recv=130759 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=134272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=127360 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=69632 expected_recv=130759 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=134272 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=69632 expected_recv=128080 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=69632 expected_recv=128080 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=69632 expected_recv=129838 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=129152 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=69632 expected_recv=129838 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=69632 expected_recv=131780 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=69632 expected_recv=128938 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=69632 expected_recv=129341 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=69632 expected_recv=131780 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=69632 expected_recv=128938 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=69632 expected_recv=129341 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=69632 expected_recv=130247 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=69632 expected_recv=130247 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=69632 expected_recv=129272 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=69632 expected_recv=130900 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=69632 expected_recv=129272 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=69632 expected_recv=130900 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=69632 expected_recv=132065 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=69632 expected_recv=132065 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=69632 expected_recv=132363 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=69632 expected_recv=132363 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=69632 expected_recv=129740 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=69632 expected_recv=129740 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=69632 expected_recv=127628 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=69632 expected_recv=129307 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=69632 expected_recv=127628 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=69632 expected_recv=131091 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=69632 expected_recv=129307 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=69632 expected_recv=125607 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=129152 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=69632 expected_recv=131091 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=69632 expected_recv=125607 actual_valid_recv=126976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=69632 expected_recv=132317 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=126976 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=69632 expected_recv=132317 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=69632 expected_recv=130024 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=69632 expected_recv=130024 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=2104389 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=106496 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=2095632 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=113792 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=2097384 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=111488 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=2094402 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=114816 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=2093999 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=115200 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=2097571 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=111616 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=2098354 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=110208 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=2095485 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=113408 peak_demand=66194 required_capacity=1.010040 + [2026-08-30 05:15:23] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 751.5 | throughput per GPU (TFLOP/s/GPU): 443.6 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101566E+01 | loss scale: 1.0 | grad norm: 45.943 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=69632 expected_recv=127740 expected_overflow=0 exact_recv_per_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132372 expected_overflow=0 exact_recv_per_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=69632 expected_recv=127740 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=69632 expected_recv=126544 expected_overflow=0 exact_recv_per_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132372 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=129152 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=69632 expected_recv=126544 actual_valid_recv=127872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=127872 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=69632 expected_recv=128453 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=69632 expected_recv=131159 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=69632 expected_recv=127044 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=69632 expected_recv=128453 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=69632 expected_recv=127044 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=69632 expected_recv=131159 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=128384 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=69632 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=69632 expected_recv=127818 expected_overflow=0 exact_recv_per_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=69632 expected_recv=129845 expected_overflow=0 exact_recv_per_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=69632 expected_recv=128993 expected_overflow=0 exact_recv_per_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=69632 expected_recv=132381 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=69632 expected_recv=127818 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=69632 expected_recv=129845 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=69632 expected_recv=128993 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=129024 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=69632 expected_recv=131397 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=69632 expected_recv=130366 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=69632 expected_recv=131397 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=69632 expected_recv=130366 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=69632 expected_recv=124893 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=69632 expected_recv=124893 actual_valid_recv=126464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=126464 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=69632 expected_recv=128516 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=69632 expected_recv=128516 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=69632 expected_recv=131436 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=69632 expected_recv=129544 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=69632 expected_recv=129543 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=69632 expected_recv=131436 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=69632 expected_recv=129544 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=69632 expected_recv=129543 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=69632 expected_recv=131326 expected_overflow=0 exact_recv_per_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=69632 expected_recv=131242 expected_overflow=0 exact_recv_per_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=69632 expected_recv=131326 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=69632 expected_recv=131242 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=69632 expected_recv=125224 expected_overflow=0 exact_recv_per_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=69632 expected_recv=131246 expected_overflow=0 exact_recv_per_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=69632 expected_recv=125224 actual_valid_recv=126592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=69632 expected_recv=131246 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=126592 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=69632 expected_recv=126016 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=69632 expected_recv=126016 actual_valid_recv=127872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=127872 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=69632 expected_recv=129735 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=69632 expected_recv=130906 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=69632 expected_recv=129735 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=69632 expected_recv=130906 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=69632 expected_recv=131006 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=69632 expected_recv=131006 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=69632 expected_recv=124541 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=69632 expected_recv=131529 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=69632 expected_recv=131963 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=69632 expected_recv=124541 actual_valid_recv=125952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=69632 expected_recv=131529 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=69632 expected_recv=131963 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=125952 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=69632 expected_recv=129277 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=69632 expected_recv=127138 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=69632 expected_recv=129277 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=69632 expected_recv=130243 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=69632 expected_recv=124446 expected_overflow=0 exact_recv_per_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=69632 expected_recv=127138 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=69632 expected_recv=130243 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=128768 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=69632 expected_recv=124446 actual_valid_recv=125824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=69632 expected_recv=131136 expected_overflow=0 exact_recv_per_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=125824 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=69632 expected_recv=131136 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=69632 expected_recv=131917 expected_overflow=0 exact_recv_per_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=69632 expected_recv=131917 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=69632 expected_recv=131731 expected_overflow=0 exact_recv_per_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=69632 expected_recv=132192 expected_overflow=0 exact_recv_per_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=69632 expected_recv=131731 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=69632 expected_recv=132192 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=69632 expected_recv=129045 expected_overflow=0 exact_recv_per_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=69632 expected_recv=129045 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=69632 expected_recv=125225 expected_overflow=0 exact_recv_per_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=69632 expected_recv=125225 actual_valid_recv=126592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=126592 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=69632 expected_recv=127058 expected_overflow=0 exact_recv_per_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=69632 expected_recv=127058 actual_valid_recv=128896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=69632 expected_recv=128228 expected_overflow=0 exact_recv_per_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=128896 slack_slots=10368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=69632 expected_recv=131373 expected_overflow=0 exact_recv_per_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=69632 expected_recv=128219 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=69632 expected_recv=128228 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=69632 expected_recv=131373 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=69632 expected_recv=127946 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=69632 expected_recv=128219 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=69632 expected_recv=130491 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=129536 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=129792 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=69632 expected_recv=127946 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=69632 expected_recv=130491 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=129536 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132283 expected_overflow=0 exact_recv_per_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132283 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=69632 expected_recv=125609 expected_overflow=0 exact_recv_per_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=69632 expected_recv=128764 expected_overflow=0 exact_recv_per_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=69632 expected_recv=125609 actual_valid_recv=126976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=69632 expected_recv=128764 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=126976 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=69632 expected_recv=131393 expected_overflow=0 exact_recv_per_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=69632 expected_recv=131393 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=69632 expected_recv=122879 expected_overflow=0 exact_recv_per_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=69632 expected_recv=122879 actual_valid_recv=124160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=124160 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=69632 expected_recv=130704 expected_overflow=0 exact_recv_per_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=69632 expected_recv=130704 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=69632 expected_recv=129825 expected_overflow=0 exact_recv_per_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=69632 expected_recv=126126 expected_overflow=0 exact_recv_per_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=69632 expected_recv=129825 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=69632 expected_recv=126126 actual_valid_recv=127360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=127360 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[56548, 41585, 85526, 67682, 70732, 86246, 43821, 72148] avg=65536.0000 max=86246 max_rank=5 min=41585 min_rank=1 imbalance_ratio=1.316010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[56548, 41585, 85526, 67682, 70732, 86246, 43821, 72148] avg=65536.0000 max=86246 max_rank=5 min=41585 min_rank=1 imbalance_ratio=1.316010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[56548, 41585, 85526, 67682, 70732, 86246, 43821, 72148] avg=65536.0000 max=86246 max_rank=5 min=41585 min_rank=1 imbalance_ratio=1.316010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[56548, 41585, 85526, 67682, 70732, 86246, 43821, 72148] avg=65536.0000 max=86246 max_rank=5 min=41585 min_rank=1 imbalance_ratio=1.316010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[56548, 41585, 85526, 67682, 70732, 86246, 43821, 72148] avg=65536.0000 max=86246 max_rank=5 min=41585 min_rank=1 imbalance_ratio=1.316010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] avg=65536.0000 max=66192 max_rank=2 min=63138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20054 max_load_reduction_pct=0.232521 moved_to_replicas=103899 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] avg=65536.0000 max=66192 max_rank=2 min=63138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20054 max_load_reduction_pct=0.232521 moved_to_replicas=103899 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[56548, 41585, 85526, 67682, 70732, 86246, 43821, 72148] avg=65536.0000 max=86246 max_rank=5 min=41585 min_rank=1 imbalance_ratio=1.316010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[56548, 41585, 85526, 67682, 70732, 86246, 43821, 72148] avg=65536.0000 max=86246 max_rank=5 min=41585 min_rank=1 imbalance_ratio=1.316010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] avg=65536.0000 max=66192 max_rank=2 min=63138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20054 max_load_reduction_pct=0.232521 moved_to_replicas=103899 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[56548, 41585, 85526, 67682, 70732, 86246, 43821, 72148] avg=65536.0000 max=86246 max_rank=5 min=41585 min_rank=1 imbalance_ratio=1.316010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] avg=65536.0000 max=66192 max_rank=2 min=63138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20054 max_load_reduction_pct=0.232521 moved_to_replicas=103899 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] avg=65536.0000 max=66192 max_rank=2 min=63138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20054 max_load_reduction_pct=0.232521 moved_to_replicas=103899 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] avg=65536.0000 max=66192 max_rank=2 min=63138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20054 max_load_reduction_pct=0.232521 moved_to_replicas=103899 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[34289, 37636, 141724, 44865, 84349, 64876, 49606, 66943] avg=65536.0000 max=141724 max_rank=2 min=34289 min_rank=0 imbalance_ratio=2.162537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] avg=65536.0000 max=66192 max_rank=2 min=63138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20054 max_load_reduction_pct=0.232521 moved_to_replicas=103899 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[34289, 37636, 141724, 44865, 84349, 64876, 49606, 66943] avg=65536.0000 max=141724 max_rank=2 min=34289 min_rank=0 imbalance_ratio=2.162537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66180, 66192, 66192, 66191, 66192, 63138, 66192] avg=65536.0000 max=66192 max_rank=2 min=63138 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20054 max_load_reduction_pct=0.232521 moved_to_replicas=103899 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[34289, 37636, 141724, 44865, 84349, 64876, 49606, 66943] avg=65536.0000 max=141724 max_rank=2 min=34289 min_rank=0 imbalance_ratio=2.162537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[34289, 37636, 141724, 44865, 84349, 64876, 49606, 66943] avg=65536.0000 max=141724 max_rank=2 min=34289 min_rank=0 imbalance_ratio=2.162537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[34289, 37636, 141724, 44865, 84349, 64876, 49606, 66943] avg=65536.0000 max=141724 max_rank=2 min=34289 min_rank=0 imbalance_ratio=2.162537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=75532 max_load_reduction_pct=0.532951 moved_to_replicas=200822 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=75532 max_load_reduction_pct=0.532951 moved_to_replicas=200822 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[34289, 37636, 141724, 44865, 84349, 64876, 49606, 66943] avg=65536.0000 max=141724 max_rank=2 min=34289 min_rank=0 imbalance_ratio=2.162537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[34289, 37636, 141724, 44865, 84349, 64876, 49606, 66943] avg=65536.0000 max=141724 max_rank=2 min=34289 min_rank=0 imbalance_ratio=2.162537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=75532 max_load_reduction_pct=0.532951 moved_to_replicas=200822 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=75532 max_load_reduction_pct=0.532951 moved_to_replicas=200822 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[34289, 37636, 141724, 44865, 84349, 64876, 49606, 66943] avg=65536.0000 max=141724 max_rank=2 min=34289 min_rank=0 imbalance_ratio=2.162537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=75532 max_load_reduction_pct=0.532951 moved_to_replicas=200822 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=75532 max_load_reduction_pct=0.532951 moved_to_replicas=200822 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[63289, 110057, 41466, 76506, 75677, 37658, 36658, 82977] avg=65536.0000 max=110057 max_rank=1 min=36658 min_rank=6 imbalance_ratio=1.679337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[63289, 110057, 41466, 76506, 75677, 37658, 36658, 82977] avg=65536.0000 max=110057 max_rank=1 min=36658 min_rank=6 imbalance_ratio=1.679337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=75532 max_load_reduction_pct=0.532951 moved_to_replicas=200822 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[63289, 110057, 41466, 76506, 75677, 37658, 36658, 82977] avg=65536.0000 max=110057 max_rank=1 min=36658 min_rank=6 imbalance_ratio=1.679337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[63289, 110057, 41466, 76506, 75677, 37658, 36658, 82977] avg=65536.0000 max=110057 max_rank=1 min=36658 min_rank=6 imbalance_ratio=1.679337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[63289, 110057, 41466, 76506, 75677, 37658, 36658, 82977] avg=65536.0000 max=110057 max_rank=1 min=36658 min_rank=6 imbalance_ratio=1.679337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66192, 62260, 66192, 64876, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62260 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=75532 max_load_reduction_pct=0.532951 moved_to_replicas=200822 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63289 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43864 max_load_reduction_pct=0.398557 moved_to_replicas=154504 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[63289, 110057, 41466, 76506, 75677, 37658, 36658, 82977] avg=65536.0000 max=110057 max_rank=1 min=36658 min_rank=6 imbalance_ratio=1.679337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63289 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43864 max_load_reduction_pct=0.398557 moved_to_replicas=154504 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63289 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43864 max_load_reduction_pct=0.398557 moved_to_replicas=154504 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63289 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43864 max_load_reduction_pct=0.398557 moved_to_replicas=154504 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63289 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43864 max_load_reduction_pct=0.398557 moved_to_replicas=154504 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[63289, 110057, 41466, 76506, 75677, 37658, 36658, 82977] avg=65536.0000 max=110057 max_rank=1 min=36658 min_rank=6 imbalance_ratio=1.679337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63289 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43864 max_load_reduction_pct=0.398557 moved_to_replicas=154504 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[63289, 110057, 41466, 76506, 75677, 37658, 36658, 82977] avg=65536.0000 max=110057 max_rank=1 min=36658 min_rank=6 imbalance_ratio=1.679337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[74407, 34772, 84064, 66214, 104939, 25070, 83846, 50976] avg=65536.0000 max=104939 max_rank=4 min=25070 min_rank=5 imbalance_ratio=1.601242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[74407, 34772, 84064, 66214, 104939, 25070, 83846, 50976] avg=65536.0000 max=104939 max_rank=4 min=25070 min_rank=5 imbalance_ratio=1.601242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63289 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43864 max_load_reduction_pct=0.398557 moved_to_replicas=154504 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[74407, 34772, 84064, 66214, 104939, 25070, 83846, 50976] avg=65536.0000 max=104939 max_rank=4 min=25070 min_rank=5 imbalance_ratio=1.601242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[74407, 34772, 84064, 66214, 104939, 25070, 83846, 50976] avg=65536.0000 max=104939 max_rank=4 min=25070 min_rank=5 imbalance_ratio=1.601242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[74407, 34772, 84064, 66214, 104939, 25070, 83846, 50976] avg=65536.0000 max=104939 max_rank=4 min=25070 min_rank=5 imbalance_ratio=1.601242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[63289, 66192, 66190, 66191, 66193, 63848, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63289 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43864 max_load_reduction_pct=0.398557 moved_to_replicas=154504 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38747 max_load_reduction_pct=0.369234 moved_to_replicas=184494 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[74407, 34772, 84064, 66214, 104939, 25070, 83846, 50976] avg=65536.0000 max=104939 max_rank=4 min=25070 min_rank=5 imbalance_ratio=1.601242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38747 max_load_reduction_pct=0.369234 moved_to_replicas=184494 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38747 max_load_reduction_pct=0.369234 moved_to_replicas=184494 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38747 max_load_reduction_pct=0.369234 moved_to_replicas=184494 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38747 max_load_reduction_pct=0.369234 moved_to_replicas=184494 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[74407, 34772, 84064, 66214, 104939, 25070, 83846, 50976] avg=65536.0000 max=104939 max_rank=4 min=25070 min_rank=5 imbalance_ratio=1.601242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38747 max_load_reduction_pct=0.369234 moved_to_replicas=184494 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[74407, 34772, 84064, 66214, 104939, 25070, 83846, 50976] avg=65536.0000 max=104939 max_rank=4 min=25070 min_rank=5 imbalance_ratio=1.601242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[26950, 86650, 76172, 38236, 144149, 43994, 63351, 44786] avg=65536.0000 max=144149 max_rank=4 min=26950 min_rank=0 imbalance_ratio=2.199539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[26950, 86650, 76172, 38236, 144149, 43994, 63351, 44786] avg=65536.0000 max=144149 max_rank=4 min=26950 min_rank=0 imbalance_ratio=2.199539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38747 max_load_reduction_pct=0.369234 moved_to_replicas=184494 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[26950, 86650, 76172, 38236, 144149, 43994, 63351, 44786] avg=65536.0000 max=144149 max_rank=4 min=26950 min_rank=0 imbalance_ratio=2.199539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[26950, 86650, 76172, 38236, 144149, 43994, 63351, 44786] avg=65536.0000 max=144149 max_rank=4 min=26950 min_rank=0 imbalance_ratio=2.199539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[26950, 86650, 76172, 38236, 144149, 43994, 63351, 44786] avg=65536.0000 max=144149 max_rank=4 min=26950 min_rank=0 imbalance_ratio=2.199539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[26950, 86650, 76172, 38236, 144149, 43994, 63351, 44786] avg=65536.0000 max=144149 max_rank=4 min=26950 min_rank=0 imbalance_ratio=2.199539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38747 max_load_reduction_pct=0.369234 moved_to_replicas=184494 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] avg=65536.0000 max=66193 max_rank=2 min=63351 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77956 max_load_reduction_pct=0.540802 moved_to_replicas=251453 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] avg=65536.0000 max=66193 max_rank=2 min=63351 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77956 max_load_reduction_pct=0.540802 moved_to_replicas=251453 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] avg=65536.0000 max=66193 max_rank=2 min=63351 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77956 max_load_reduction_pct=0.540802 moved_to_replicas=251453 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] avg=65536.0000 max=66193 max_rank=2 min=63351 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77956 max_load_reduction_pct=0.540802 moved_to_replicas=251453 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] avg=65536.0000 max=66193 max_rank=2 min=63351 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77956 max_load_reduction_pct=0.540802 moved_to_replicas=251453 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[26950, 86650, 76172, 38236, 144149, 43994, 63351, 44786] avg=65536.0000 max=144149 max_rank=4 min=26950 min_rank=0 imbalance_ratio=2.199539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] avg=65536.0000 max=66193 max_rank=2 min=63351 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77956 max_load_reduction_pct=0.540802 moved_to_replicas=251453 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[26950, 86650, 76172, 38236, 144149, 43994, 63351, 44786] avg=65536.0000 max=144149 max_rank=4 min=26950 min_rank=0 imbalance_ratio=2.199539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[33702, 92391, 103149, 59660, 73430, 48618, 20638, 92700] avg=65536.0000 max=103149 max_rank=2 min=20638 min_rank=6 imbalance_ratio=1.573929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[33702, 92391, 103149, 59660, 73430, 48618, 20638, 92700] avg=65536.0000 max=103149 max_rank=2 min=20638 min_rank=6 imbalance_ratio=1.573929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[33702, 92391, 103149, 59660, 73430, 48618, 20638, 92700] avg=65536.0000 max=103149 max_rank=2 min=20638 min_rank=6 imbalance_ratio=1.573929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] avg=65536.0000 max=66193 max_rank=2 min=63351 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77956 max_load_reduction_pct=0.540802 moved_to_replicas=251453 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[33702, 92391, 103149, 59660, 73430, 48618, 20638, 92700] avg=65536.0000 max=103149 max_rank=2 min=20638 min_rank=6 imbalance_ratio=1.573929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[33702, 92391, 103149, 59660, 73430, 48618, 20638, 92700] avg=65536.0000 max=103149 max_rank=2 min=20638 min_rank=6 imbalance_ratio=1.573929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[33702, 92391, 103149, 59660, 73430, 48618, 20638, 92700] avg=65536.0000 max=103149 max_rank=2 min=20638 min_rank=6 imbalance_ratio=1.573929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[66191, 66192, 66193, 66192, 66192, 64733, 63351, 65244] avg=65536.0000 max=66193 max_rank=2 min=63351 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77956 max_load_reduction_pct=0.540802 moved_to_replicas=251453 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] avg=65536.0000 max=66193 max_rank=1 min=62085 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36956 max_load_reduction_pct=0.358278 moved_to_replicas=196334 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] avg=65536.0000 max=66193 max_rank=1 min=62085 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36956 max_load_reduction_pct=0.358278 moved_to_replicas=196334 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] avg=65536.0000 max=66193 max_rank=1 min=62085 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36956 max_load_reduction_pct=0.358278 moved_to_replicas=196334 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] avg=65536.0000 max=66193 max_rank=1 min=62085 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36956 max_load_reduction_pct=0.358278 moved_to_replicas=196334 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] avg=65536.0000 max=66193 max_rank=1 min=62085 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36956 max_load_reduction_pct=0.358278 moved_to_replicas=196334 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] avg=65536.0000 max=66193 max_rank=1 min=62085 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36956 max_load_reduction_pct=0.358278 moved_to_replicas=196334 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[33702, 92391, 103149, 59660, 73430, 48618, 20638, 92700] avg=65536.0000 max=103149 max_rank=2 min=20638 min_rank=6 imbalance_ratio=1.573929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[119625, 3196, 5070, 68462, 35641, 78661, 110194, 103439] avg=65536.0000 max=119625 max_rank=0 min=3196 min_rank=1 imbalance_ratio=1.825333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[33702, 92391, 103149, 59660, 73430, 48618, 20638, 92700] avg=65536.0000 max=103149 max_rank=2 min=20638 min_rank=6 imbalance_ratio=1.573929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[119625, 3196, 5070, 68462, 35641, 78661, 110194, 103439] avg=65536.0000 max=119625 max_rank=0 min=3196 min_rank=1 imbalance_ratio=1.825333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[119625, 3196, 5070, 68462, 35641, 78661, 110194, 103439] avg=65536.0000 max=119625 max_rank=0 min=3196 min_rank=1 imbalance_ratio=1.825333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] avg=65536.0000 max=66193 max_rank=1 min=62085 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36956 max_load_reduction_pct=0.358278 moved_to_replicas=196334 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[119625, 3196, 5070, 68462, 35641, 78661, 110194, 103439] avg=65536.0000 max=119625 max_rank=0 min=3196 min_rank=1 imbalance_ratio=1.825333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[119625, 3196, 5070, 68462, 35641, 78661, 110194, 103439] avg=65536.0000 max=119625 max_rank=0 min=3196 min_rank=1 imbalance_ratio=1.825333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[119625, 3196, 5070, 68462, 35641, 78661, 110194, 103439] avg=65536.0000 max=119625 max_rank=0 min=3196 min_rank=1 imbalance_ratio=1.825333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53432 max_load_reduction_pct=0.446662 moved_to_replicas=286518 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[65050, 66193, 66192, 66193, 66191, 66193, 62085, 66191] avg=65536.0000 max=66193 max_rank=1 min=62085 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36956 max_load_reduction_pct=0.358278 moved_to_replicas=196334 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53432 max_load_reduction_pct=0.446662 moved_to_replicas=286518 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53432 max_load_reduction_pct=0.446662 moved_to_replicas=286518 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53432 max_load_reduction_pct=0.446662 moved_to_replicas=286518 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53432 max_load_reduction_pct=0.446662 moved_to_replicas=286518 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53432 max_load_reduction_pct=0.446662 moved_to_replicas=286518 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[119625, 3196, 5070, 68462, 35641, 78661, 110194, 103439] avg=65536.0000 max=119625 max_rank=0 min=3196 min_rank=1 imbalance_ratio=1.825333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[100150, 2639, 76081, 58484, 125674, 47528, 5807, 107925] avg=65536.0000 max=125674 max_rank=4 min=2639 min_rank=1 imbalance_ratio=1.917633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[119625, 3196, 5070, 68462, 35641, 78661, 110194, 103439] avg=65536.0000 max=119625 max_rank=0 min=3196 min_rank=1 imbalance_ratio=1.825333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[100150, 2639, 76081, 58484, 125674, 47528, 5807, 107925] avg=65536.0000 max=125674 max_rank=4 min=2639 min_rank=1 imbalance_ratio=1.917633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[100150, 2639, 76081, 58484, 125674, 47528, 5807, 107925] avg=65536.0000 max=125674 max_rank=4 min=2639 min_rank=1 imbalance_ratio=1.917633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53432 max_load_reduction_pct=0.446662 moved_to_replicas=286518 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[100150, 2639, 76081, 58484, 125674, 47528, 5807, 107925] avg=65536.0000 max=125674 max_rank=4 min=2639 min_rank=1 imbalance_ratio=1.917633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[100150, 2639, 76081, 58484, 125674, 47528, 5807, 107925] avg=65536.0000 max=125674 max_rank=4 min=2639 min_rank=1 imbalance_ratio=1.917633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[100150, 2639, 76081, 58484, 125674, 47528, 5807, 107925] avg=65536.0000 max=125674 max_rank=4 min=2639 min_rank=1 imbalance_ratio=1.917633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] avg=65536.0000 max=66193 max_rank=7 min=61800 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=59481 max_load_reduction_pct=0.473296 moved_to_replicas=271085 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66192, 60944, 66192, 66192, 66191, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53432 max_load_reduction_pct=0.446662 moved_to_replicas=286518 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] avg=65536.0000 max=66193 max_rank=7 min=61800 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=59481 max_load_reduction_pct=0.473296 moved_to_replicas=271085 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] avg=65536.0000 max=66193 max_rank=7 min=61800 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=59481 max_load_reduction_pct=0.473296 moved_to_replicas=271085 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] avg=65536.0000 max=66193 max_rank=7 min=61800 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=59481 max_load_reduction_pct=0.473296 moved_to_replicas=271085 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] avg=65536.0000 max=66193 max_rank=7 min=61800 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=59481 max_load_reduction_pct=0.473296 moved_to_replicas=271085 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] avg=65536.0000 max=66193 max_rank=7 min=61800 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=59481 max_load_reduction_pct=0.473296 moved_to_replicas=271085 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[100150, 2639, 76081, 58484, 125674, 47528, 5807, 107925] avg=65536.0000 max=125674 max_rank=4 min=2639 min_rank=1 imbalance_ratio=1.917633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[85009, 31019, 82973, 92375, 79353, 83955, 57571, 12033] avg=65536.0000 max=92375 max_rank=3 min=12033 min_rank=7 imbalance_ratio=1.409531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[85009, 31019, 82973, 92375, 79353, 83955, 57571, 12033] avg=65536.0000 max=92375 max_rank=3 min=12033 min_rank=7 imbalance_ratio=1.409531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[85009, 31019, 82973, 92375, 79353, 83955, 57571, 12033] avg=65536.0000 max=92375 max_rank=3 min=12033 min_rank=7 imbalance_ratio=1.409531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[100150, 2639, 76081, 58484, 125674, 47528, 5807, 107925] avg=65536.0000 max=125674 max_rank=4 min=2639 min_rank=1 imbalance_ratio=1.917633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] avg=65536.0000 max=66193 max_rank=7 min=61800 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=59481 max_load_reduction_pct=0.473296 moved_to_replicas=271085 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[85009, 31019, 82973, 92375, 79353, 83955, 57571, 12033] avg=65536.0000 max=92375 max_rank=3 min=12033 min_rank=7 imbalance_ratio=1.409531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26182 max_load_reduction_pct=0.283432 moved_to_replicas=199410 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[85009, 31019, 82973, 92375, 79353, 83955, 57571, 12033] avg=65536.0000 max=92375 max_rank=3 min=12033 min_rank=7 imbalance_ratio=1.409531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26182 max_load_reduction_pct=0.283432 moved_to_replicas=199410 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26182 max_load_reduction_pct=0.283432 moved_to_replicas=199410 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[85009, 31019, 82973, 92375, 79353, 83955, 57571, 12033] avg=65536.0000 max=92375 max_rank=3 min=12033 min_rank=7 imbalance_ratio=1.409531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66190, 66192, 66192, 66191, 61800, 65338, 66193] avg=65536.0000 max=66193 max_rank=7 min=61800 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=59481 max_load_reduction_pct=0.473296 moved_to_replicas=271085 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26182 max_load_reduction_pct=0.283432 moved_to_replicas=199410 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26182 max_load_reduction_pct=0.283432 moved_to_replicas=199410 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[85009, 31019, 82973, 92375, 79353, 83955, 57571, 12033] avg=65536.0000 max=92375 max_rank=3 min=12033 min_rank=7 imbalance_ratio=1.409531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26182 max_load_reduction_pct=0.283432 moved_to_replicas=199410 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[65908, 36182, 61246, 118273, 109237, 33836, 33880, 65726] avg=65536.0000 max=118273 max_rank=3 min=33836 min_rank=5 imbalance_ratio=1.804703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[65908, 36182, 61246, 118273, 109237, 33836, 33880, 65726] avg=65536.0000 max=118273 max_rank=3 min=33836 min_rank=5 imbalance_ratio=1.804703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[65908, 36182, 61246, 118273, 109237, 33836, 33880, 65726] avg=65536.0000 max=118273 max_rank=3 min=33836 min_rank=5 imbalance_ratio=1.804703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[85009, 31019, 82973, 92375, 79353, 83955, 57571, 12033] avg=65536.0000 max=92375 max_rank=3 min=12033 min_rank=7 imbalance_ratio=1.409531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26182 max_load_reduction_pct=0.283432 moved_to_replicas=199410 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[65908, 36182, 61246, 118273, 109237, 33836, 33880, 65726] avg=65536.0000 max=118273 max_rank=3 min=33836 min_rank=5 imbalance_ratio=1.804703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] avg=65536.0000 max=66193 max_rank=6 min=61886 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=52080 max_load_reduction_pct=0.440337 moved_to_replicas=197546 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[65908, 36182, 61246, 118273, 109237, 33836, 33880, 65726] avg=65536.0000 max=118273 max_rank=3 min=33836 min_rank=5 imbalance_ratio=1.804703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] avg=65536.0000 max=66193 max_rank=6 min=61886 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=52080 max_load_reduction_pct=0.440337 moved_to_replicas=197546 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] avg=65536.0000 max=66193 max_rank=6 min=61886 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=52080 max_load_reduction_pct=0.440337 moved_to_replicas=197546 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66193, 66192, 60944, 66191] avg=65536.0000 max=66193 max_rank=0 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26182 max_load_reduction_pct=0.283432 moved_to_replicas=199410 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[65908, 36182, 61246, 118273, 109237, 33836, 33880, 65726] avg=65536.0000 max=118273 max_rank=3 min=33836 min_rank=5 imbalance_ratio=1.804703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] avg=65536.0000 max=66193 max_rank=6 min=61886 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=52080 max_load_reduction_pct=0.440337 moved_to_replicas=197546 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[65908, 36182, 61246, 118273, 109237, 33836, 33880, 65726] avg=65536.0000 max=118273 max_rank=3 min=33836 min_rank=5 imbalance_ratio=1.804703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] avg=65536.0000 max=66193 max_rank=6 min=61886 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=52080 max_load_reduction_pct=0.440337 moved_to_replicas=197546 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] avg=65536.0000 max=66193 max_rank=6 min=61886 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=52080 max_load_reduction_pct=0.440337 moved_to_replicas=197546 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[69235, 31746, 84079, 32218, 106044, 39227, 118709, 43030] avg=65536.0000 max=118709 max_rank=6 min=31746 min_rank=1 imbalance_ratio=1.811356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[69235, 31746, 84079, 32218, 106044, 39227, 118709, 43030] avg=65536.0000 max=118709 max_rank=6 min=31746 min_rank=1 imbalance_ratio=1.811356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[69235, 31746, 84079, 32218, 106044, 39227, 118709, 43030] avg=65536.0000 max=118709 max_rank=6 min=31746 min_rank=1 imbalance_ratio=1.811356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[65908, 36182, 61246, 118273, 109237, 33836, 33880, 65726] avg=65536.0000 max=118273 max_rank=3 min=33836 min_rank=5 imbalance_ratio=1.804703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] avg=65536.0000 max=66193 max_rank=6 min=61886 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=52080 max_load_reduction_pct=0.440337 moved_to_replicas=197546 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[69235, 31746, 84079, 32218, 106044, 39227, 118709, 43030] avg=65536.0000 max=118709 max_rank=6 min=31746 min_rank=1 imbalance_ratio=1.811356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] avg=65536.0000 max=66193 max_rank=4 min=62852 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52516 max_load_reduction_pct=0.442393 moved_to_replicas=231156 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[69235, 31746, 84079, 32218, 106044, 39227, 118709, 43030] avg=65536.0000 max=118709 max_rank=6 min=31746 min_rank=1 imbalance_ratio=1.811356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] avg=65536.0000 max=66193 max_rank=4 min=62852 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52516 max_load_reduction_pct=0.442393 moved_to_replicas=231156 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[69235, 31746, 84079, 32218, 106044, 39227, 118709, 43030] avg=65536.0000 max=118709 max_rank=6 min=31746 min_rank=1 imbalance_ratio=1.811356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] avg=65536.0000 max=66193 max_rank=4 min=62852 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52516 max_load_reduction_pct=0.442393 moved_to_replicas=231156 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[65908, 66192, 61886, 66192, 66191, 66000, 66193, 65726] avg=65536.0000 max=66193 max_rank=6 min=61886 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=52080 max_load_reduction_pct=0.440337 moved_to_replicas=197546 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] avg=65536.0000 max=66193 max_rank=4 min=62852 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52516 max_load_reduction_pct=0.442393 moved_to_replicas=231156 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[69235, 31746, 84079, 32218, 106044, 39227, 118709, 43030] avg=65536.0000 max=118709 max_rank=6 min=31746 min_rank=1 imbalance_ratio=1.811356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] avg=65536.0000 max=66193 max_rank=4 min=62852 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52516 max_load_reduction_pct=0.442393 moved_to_replicas=231156 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] avg=65536.0000 max=66193 max_rank=4 min=62852 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52516 max_load_reduction_pct=0.442393 moved_to_replicas=231156 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[81411, 20637, 93980, 91303, 69680, 46473, 7949, 112855] avg=65536.0000 max=112855 max_rank=7 min=7949 min_rank=6 imbalance_ratio=1.722031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[81411, 20637, 93980, 91303, 69680, 46473, 7949, 112855] avg=65536.0000 max=112855 max_rank=7 min=7949 min_rank=6 imbalance_ratio=1.722031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] avg=65536.0000 max=66193 max_rank=4 min=62852 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52516 max_load_reduction_pct=0.442393 moved_to_replicas=231156 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[81411, 20637, 93980, 91303, 69680, 46473, 7949, 112855] avg=65536.0000 max=112855 max_rank=7 min=7949 min_rank=6 imbalance_ratio=1.722031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[69235, 31746, 84079, 32218, 106044, 39227, 118709, 43030] avg=65536.0000 max=118709 max_rank=6 min=31746 min_rank=1 imbalance_ratio=1.811356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[81411, 20637, 93980, 91303, 69680, 46473, 7949, 112855] avg=65536.0000 max=112855 max_rank=7 min=7949 min_rank=6 imbalance_ratio=1.722031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] avg=65536.0000 max=66192 max_rank=0 min=64013 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46663 max_load_reduction_pct=0.413477 moved_to_replicas=240553 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[81411, 20637, 93980, 91303, 69680, 46473, 7949, 112855] avg=65536.0000 max=112855 max_rank=7 min=7949 min_rank=6 imbalance_ratio=1.722031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] avg=65536.0000 max=66192 max_rank=0 min=64013 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46663 max_load_reduction_pct=0.413477 moved_to_replicas=240553 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[81411, 20637, 93980, 91303, 69680, 46473, 7949, 112855] avg=65536.0000 max=112855 max_rank=7 min=7949 min_rank=6 imbalance_ratio=1.722031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] avg=65536.0000 max=66192 max_rank=0 min=64013 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46663 max_load_reduction_pct=0.413477 moved_to_replicas=240553 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64282, 66192, 66192, 66193, 66193, 66192, 62852] avg=65536.0000 max=66193 max_rank=4 min=62852 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52516 max_load_reduction_pct=0.442393 moved_to_replicas=231156 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] avg=65536.0000 max=66192 max_rank=0 min=64013 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46663 max_load_reduction_pct=0.413477 moved_to_replicas=240553 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[81411, 20637, 93980, 91303, 69680, 46473, 7949, 112855] avg=65536.0000 max=112855 max_rank=7 min=7949 min_rank=6 imbalance_ratio=1.722031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] avg=65536.0000 max=66192 max_rank=0 min=64013 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46663 max_load_reduction_pct=0.413477 moved_to_replicas=240553 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] avg=65536.0000 max=66192 max_rank=0 min=64013 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46663 max_load_reduction_pct=0.413477 moved_to_replicas=240553 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[57898, 116010, 14165, 885, 74718, 70827, 112961, 76824] avg=65536.0000 max=116010 max_rank=1 min=885 min_rank=3 imbalance_ratio=1.770172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[57898, 116010, 14165, 885, 74718, 70827, 112961, 76824] avg=65536.0000 max=116010 max_rank=1 min=885 min_rank=3 imbalance_ratio=1.770172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] avg=65536.0000 max=66192 max_rank=0 min=64013 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46663 max_load_reduction_pct=0.413477 moved_to_replicas=240553 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[57898, 116010, 14165, 885, 74718, 70827, 112961, 76824] avg=65536.0000 max=116010 max_rank=1 min=885 min_rank=3 imbalance_ratio=1.770172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[81411, 20637, 93980, 91303, 69680, 46473, 7949, 112855] avg=65536.0000 max=112855 max_rank=7 min=7949 min_rank=6 imbalance_ratio=1.722031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62995 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49817 max_load_reduction_pct=0.429420 moved_to_replicas=243284 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62995 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49817 max_load_reduction_pct=0.429420 moved_to_replicas=243284 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[57898, 116010, 14165, 885, 74718, 70827, 112961, 76824] avg=65536.0000 max=116010 max_rank=1 min=885 min_rank=3 imbalance_ratio=1.770172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[57898, 116010, 14165, 885, 74718, 70827, 112961, 76824] avg=65536.0000 max=116010 max_rank=1 min=885 min_rank=3 imbalance_ratio=1.770172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[57898, 116010, 14165, 885, 74718, 70827, 112961, 76824] avg=65536.0000 max=116010 max_rank=1 min=885 min_rank=3 imbalance_ratio=1.770172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62995 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49817 max_load_reduction_pct=0.429420 moved_to_replicas=243284 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64013, 66192, 66192, 66192, 65180, 64135, 66192] avg=65536.0000 max=66192 max_rank=0 min=64013 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46663 max_load_reduction_pct=0.413477 moved_to_replicas=240553 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[57898, 116010, 14165, 885, 74718, 70827, 112961, 76824] avg=65536.0000 max=116010 max_rank=1 min=885 min_rank=3 imbalance_ratio=1.770172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62995 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49817 max_load_reduction_pct=0.429420 moved_to_replicas=243284 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62995 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49817 max_load_reduction_pct=0.429420 moved_to_replicas=243284 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62995 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49817 max_load_reduction_pct=0.429420 moved_to_replicas=243284 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[91621, 33087, 94949, 12083, 64329, 33076, 115714, 79429] avg=65536.0000 max=115714 max_rank=6 min=12083 min_rank=3 imbalance_ratio=1.765656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[91621, 33087, 94949, 12083, 64329, 33076, 115714, 79429] avg=65536.0000 max=115714 max_rank=6 min=12083 min_rank=3 imbalance_ratio=1.765656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62995 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49817 max_load_reduction_pct=0.429420 moved_to_replicas=243284 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[91621, 33087, 94949, 12083, 64329, 33076, 115714, 79429] avg=65536.0000 max=115714 max_rank=6 min=12083 min_rank=3 imbalance_ratio=1.765656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[57898, 116010, 14165, 885, 74718, 70827, 112961, 76824] avg=65536.0000 max=116010 max_rank=1 min=885 min_rank=3 imbalance_ratio=1.770172 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62807 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=49522 max_load_reduction_pct=0.427969 moved_to_replicas=257027 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62807 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=49522 max_load_reduction_pct=0.427969 moved_to_replicas=257027 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62807 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=49522 max_load_reduction_pct=0.427969 moved_to_replicas=257027 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[91621, 33087, 94949, 12083, 64329, 33076, 115714, 79429] avg=65536.0000 max=115714 max_rank=6 min=12083 min_rank=3 imbalance_ratio=1.765656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[91621, 33087, 94949, 12083, 64329, 33076, 115714, 79429] avg=65536.0000 max=115714 max_rank=6 min=12083 min_rank=3 imbalance_ratio=1.765656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[91621, 33087, 94949, 12083, 64329, 33076, 115714, 79429] avg=65536.0000 max=115714 max_rank=6 min=12083 min_rank=3 imbalance_ratio=1.765656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62995, 64141, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=62995 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49817 max_load_reduction_pct=0.429420 moved_to_replicas=243284 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[91621, 33087, 94949, 12083, 64329, 33076, 115714, 79429] avg=65536.0000 max=115714 max_rank=6 min=12083 min_rank=3 imbalance_ratio=1.765656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62807 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=49522 max_load_reduction_pct=0.427969 moved_to_replicas=257027 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[74528, 50705, 92121, 36827, 38537, 88504, 103636, 39430] avg=65536.0000 max=103636 max_rank=6 min=36827 min_rank=3 imbalance_ratio=1.581360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62807 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=49522 max_load_reduction_pct=0.427969 moved_to_replicas=257027 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[74528, 50705, 92121, 36827, 38537, 88504, 103636, 39430] avg=65536.0000 max=103636 max_rank=6 min=36827 min_rank=3 imbalance_ratio=1.581360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62807 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=49522 max_load_reduction_pct=0.427969 moved_to_replicas=257027 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62807 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=49522 max_load_reduction_pct=0.427969 moved_to_replicas=257027 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[74528, 50705, 92121, 36827, 38537, 88504, 103636, 39430] avg=65536.0000 max=103636 max_rank=6 min=36827 min_rank=3 imbalance_ratio=1.581360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[91621, 33087, 94949, 12083, 64329, 33076, 115714, 79429] avg=65536.0000 max=115714 max_rank=6 min=12083 min_rank=3 imbalance_ratio=1.765656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=0 min=60943 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=37443 max_load_reduction_pct=0.361293 moved_to_replicas=190591 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=0 min=60943 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=37443 max_load_reduction_pct=0.361293 moved_to_replicas=190591 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=0 min=60943 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=37443 max_load_reduction_pct=0.361293 moved_to_replicas=190591 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[74528, 50705, 92121, 36827, 38537, 88504, 103636, 39430] avg=65536.0000 max=103636 max_rank=6 min=36827 min_rank=3 imbalance_ratio=1.581360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[74528, 50705, 92121, 36827, 38537, 88504, 103636, 39430] avg=65536.0000 max=103636 max_rank=6 min=36827 min_rank=3 imbalance_ratio=1.581360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[74528, 50705, 92121, 36827, 38537, 88504, 103636, 39430] avg=65536.0000 max=103636 max_rank=6 min=36827 min_rank=3 imbalance_ratio=1.581360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[74528, 50705, 92121, 36827, 38537, 88504, 103636, 39430] avg=65536.0000 max=103636 max_rank=6 min=36827 min_rank=3 imbalance_ratio=1.581360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[66192, 62807, 66192, 66192, 64329, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62807 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=49522 max_load_reduction_pct=0.427969 moved_to_replicas=257027 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=0 min=60943 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=37443 max_load_reduction_pct=0.361293 moved_to_replicas=190591 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=0 min=60943 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=37443 max_load_reduction_pct=0.361293 moved_to_replicas=190591 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[23453, 55156, 135829, 74281, 57104, 36463, 62128, 79874] avg=65536.0000 max=135829 max_rank=2 min=23453 min_rank=0 imbalance_ratio=2.072586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[23453, 55156, 135829, 74281, 57104, 36463, 62128, 79874] avg=65536.0000 max=135829 max_rank=2 min=23453 min_rank=0 imbalance_ratio=2.072586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=0 min=60943 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=37443 max_load_reduction_pct=0.361293 moved_to_replicas=190591 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[23453, 55156, 135829, 74281, 57104, 36463, 62128, 79874] avg=65536.0000 max=135829 max_rank=2 min=23453 min_rank=0 imbalance_ratio=2.072586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=0 min=60943 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=37443 max_load_reduction_pct=0.361293 moved_to_replicas=190591 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[74528, 50705, 92121, 36827, 38537, 88504, 103636, 39430] avg=65536.0000 max=103636 max_rank=6 min=36827 min_rank=3 imbalance_ratio=1.581360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] avg=65536.0000 max=66193 max_rank=2 min=62624 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69636 max_load_reduction_pct=0.512674 moved_to_replicas=182065 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] avg=65536.0000 max=66193 max_rank=2 min=62624 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69636 max_load_reduction_pct=0.512674 moved_to_replicas=182065 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] avg=65536.0000 max=66193 max_rank=2 min=62624 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69636 max_load_reduction_pct=0.512674 moved_to_replicas=182065 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[23453, 55156, 135829, 74281, 57104, 36463, 62128, 79874] avg=65536.0000 max=135829 max_rank=2 min=23453 min_rank=0 imbalance_ratio=2.072586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[23453, 55156, 135829, 74281, 57104, 36463, 62128, 79874] avg=65536.0000 max=135829 max_rank=2 min=23453 min_rank=0 imbalance_ratio=2.072586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[66193, 60943, 66191, 66192, 66193, 66192, 66191, 66193] avg=65536.0000 max=66193 max_rank=0 min=60943 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=37443 max_load_reduction_pct=0.361293 moved_to_replicas=190591 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[23453, 55156, 135829, 74281, 57104, 36463, 62128, 79874] avg=65536.0000 max=135829 max_rank=2 min=23453 min_rank=0 imbalance_ratio=2.072586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[23453, 55156, 135829, 74281, 57104, 36463, 62128, 79874] avg=65536.0000 max=135829 max_rank=2 min=23453 min_rank=0 imbalance_ratio=2.072586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] avg=65536.0000 max=66193 max_rank=2 min=62624 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69636 max_load_reduction_pct=0.512674 moved_to_replicas=182065 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] avg=65536.0000 max=66193 max_rank=2 min=62624 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69636 max_load_reduction_pct=0.512674 moved_to_replicas=182065 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] avg=65536.0000 max=66193 max_rank=2 min=62624 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69636 max_load_reduction_pct=0.512674 moved_to_replicas=182065 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] avg=65536.0000 max=66193 max_rank=2 min=62624 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69636 max_load_reduction_pct=0.512674 moved_to_replicas=182065 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[23453, 55156, 135829, 74281, 57104, 36463, 62128, 79874] avg=65536.0000 max=135829 max_rank=2 min=23453 min_rank=0 imbalance_ratio=2.072586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[66192, 62624, 66193, 66191, 66192, 66192, 64512, 66192] avg=65536.0000 max=66193 max_rank=2 min=62624 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69636 max_load_reduction_pct=0.512674 moved_to_replicas=182065 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=2099841 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=108288 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=2089196 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=118144 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=2105025 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=104320 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=2108071 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=100224 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=2075251 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=131584 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=2104649 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=102656 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=2099669 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=108416 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=2095514 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=111744 peak_demand=66193 required_capacity=1.010025 + [2026-08-30 05:15:24] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 543.2 | throughput per GPU (TFLOP/s/GPU): 613.8 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.052046E+01 | loss scale: 1.0 | grad norm: 215.188 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=69632 expected_recv=129285 expected_overflow=0 exact_recv_per_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=69632 expected_recv=127349 expected_overflow=0 exact_recv_per_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=69632 expected_recv=129285 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=69632 expected_recv=127349 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=69632 expected_recv=130020 expected_overflow=0 exact_recv_per_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=128384 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=69632 expected_recv=130020 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=69632 expected_recv=130822 expected_overflow=0 exact_recv_per_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=69632 expected_recv=131053 expected_overflow=0 exact_recv_per_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=69632 expected_recv=130822 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=69632 expected_recv=127940 expected_overflow=0 exact_recv_per_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=69632 expected_recv=130958 expected_overflow=0 exact_recv_per_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=69632 expected_recv=131053 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=69632 expected_recv=130650 expected_overflow=0 exact_recv_per_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=69632 expected_recv=127940 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=69632 expected_recv=130958 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=69632 expected_recv=130650 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=129280 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=69632 expected_recv=131101 expected_overflow=0 exact_recv_per_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=69632 expected_recv=131101 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=69632 expected_recv=129014 expected_overflow=0 exact_recv_per_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=69632 expected_recv=129014 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=69632 expected_recv=129433 expected_overflow=0 exact_recv_per_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=69632 expected_recv=129491 expected_overflow=0 exact_recv_per_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=69632 expected_recv=129433 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=69632 expected_recv=129491 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=69632 expected_recv=131708 expected_overflow=0 exact_recv_per_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=69632 expected_recv=131708 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=69632 expected_recv=131068 expected_overflow=0 exact_recv_per_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=69632 expected_recv=131400 expected_overflow=0 exact_recv_per_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=69632 expected_recv=126096 expected_overflow=0 exact_recv_per_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=69632 expected_recv=131068 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=69632 expected_recv=131151 expected_overflow=0 exact_recv_per_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=69632 expected_recv=131400 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=69632 expected_recv=126096 actual_valid_recv=127488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=69632 expected_recv=131151 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=127488 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=69632 expected_recv=131546 expected_overflow=0 exact_recv_per_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=69632 expected_recv=131546 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=69632 expected_recv=127910 expected_overflow=0 exact_recv_per_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=69632 expected_recv=131930 expected_overflow=0 exact_recv_per_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=69632 expected_recv=127910 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=69632 expected_recv=131930 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=69632 expected_recv=127655 expected_overflow=0 exact_recv_per_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=129664 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=69632 expected_recv=131913 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=69632 expected_recv=127655 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=129408 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=69632 expected_recv=131913 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=69632 expected_recv=127899 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=69632 expected_recv=127541 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=69632 expected_recv=131685 expected_overflow=0 exact_recv_per_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=69632 expected_recv=127899 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=69632 expected_recv=130998 expected_overflow=0 exact_recv_per_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=69632 expected_recv=127541 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=129536 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=69632 expected_recv=131685 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=129024 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=69632 expected_recv=131378 expected_overflow=0 exact_recv_per_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=69632 expected_recv=130998 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=69632 expected_recv=131378 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=69632 expected_recv=124280 expected_overflow=0 exact_recv_per_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=69632 expected_recv=124280 actual_valid_recv=125824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=125824 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=69632 expected_recv=130755 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=69632 expected_recv=130755 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=69632 expected_recv=131466 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=69632 expected_recv=124436 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=69632 expected_recv=131466 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=69632 expected_recv=124436 actual_valid_recv=126080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=126080 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=69632 expected_recv=128270 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=69632 expected_recv=129552 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=69632 expected_recv=128270 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=69632 expected_recv=131110 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=69632 expected_recv=129552 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=129792 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=69632 expected_recv=131110 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=69632 expected_recv=130108 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=69632 expected_recv=130108 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=69632 expected_recv=131429 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=69632 expected_recv=132319 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=69632 expected_recv=131429 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=69632 expected_recv=132319 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=69632 expected_recv=126196 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=69632 expected_recv=129646 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=69632 expected_recv=131835 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=69632 expected_recv=126196 actual_valid_recv=127488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=69632 expected_recv=129646 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=127488 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=69632 expected_recv=131835 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=69632 expected_recv=129777 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=69632 expected_recv=131316 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=69632 expected_recv=129777 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=69632 expected_recv=131316 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=69632 expected_recv=131054 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=69632 expected_recv=130694 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=69632 expected_recv=128581 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=69632 expected_recv=131054 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=69632 expected_recv=130694 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=69632 expected_recv=128581 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=69632 expected_recv=127834 expected_overflow=0 exact_recv_per_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=69632 expected_recv=127834 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=69632 expected_recv=130464 expected_overflow=0 exact_recv_per_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=134144 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=129408 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=69632 expected_recv=130464 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=69632 expected_recv=129492 expected_overflow=0 exact_recv_per_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=69632 expected_recv=131248 expected_overflow=0 exact_recv_per_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=69632 expected_recv=129492 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=69632 expected_recv=131248 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=69632 expected_recv=127620 expected_overflow=0 exact_recv_per_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=69632 expected_recv=131689 expected_overflow=0 exact_recv_per_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=69632 expected_recv=127620 actual_valid_recv=128896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=69632 expected_recv=130556 expected_overflow=0 exact_recv_per_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=128896 slack_slots=10368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=69632 expected_recv=131689 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=69632 expected_recv=130556 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=69632 expected_recv=129174 expected_overflow=0 exact_recv_per_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=69632 expected_recv=130599 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=69632 expected_recv=129673 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=69632 expected_recv=129174 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=69632 expected_recv=130599 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=69632 expected_recv=129673 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=69632 expected_recv=128861 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=69632 expected_recv=128861 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=69632 expected_recv=129907 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=69632 expected_recv=129907 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=69632 expected_recv=130116 expected_overflow=0 exact_recv_per_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=69632 expected_recv=129586 expected_overflow=0 exact_recv_per_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=69632 expected_recv=130116 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=69632 expected_recv=129586 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=69632 expected_recv=129998 expected_overflow=0 exact_recv_per_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=69632 expected_recv=131305 expected_overflow=0 exact_recv_per_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=69632 expected_recv=129998 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=69632 expected_recv=131305 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=69632 expected_recv=130421 expected_overflow=0 exact_recv_per_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=69632 expected_recv=129225 expected_overflow=0 exact_recv_per_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=69632 expected_recv=130421 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=69632 expected_recv=129225 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=69632 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=69632 expected_recv=129902 expected_overflow=0 exact_recv_per_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=69632 expected_recv=130296 expected_overflow=0 exact_recv_per_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=69632 expected_recv=132381 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=69632 expected_recv=129902 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=69632 expected_recv=130296 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=69632 expected_recv=129620 expected_overflow=0 exact_recv_per_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=69632 expected_recv=129620 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[47035, 39248, 99656, 57229, 63828, 96266, 49528, 71498] avg=65536.0000 max=99656 max_rank=2 min=39248 min_rank=1 imbalance_ratio=1.520630 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[47035, 39248, 99656, 57229, 63828, 96266, 49528, 71498] avg=65536.0000 max=99656 max_rank=2 min=39248 min_rank=1 imbalance_ratio=1.520630 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[47035, 39248, 99656, 57229, 63828, 96266, 49528, 71498] avg=65536.0000 max=99656 max_rank=2 min=39248 min_rank=1 imbalance_ratio=1.520630 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[47035, 39248, 99656, 57229, 63828, 96266, 49528, 71498] avg=65536.0000 max=99656 max_rank=2 min=39248 min_rank=1 imbalance_ratio=1.520630 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[47035, 39248, 99656, 57229, 63828, 96266, 49528, 71498] avg=65536.0000 max=99656 max_rank=2 min=39248 min_rank=1 imbalance_ratio=1.520630 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] avg=65536.0000 max=66193 max_rank=2 min=63667 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33463 max_load_reduction_pct=0.335785 moved_to_replicas=135239 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[47035, 39248, 99656, 57229, 63828, 96266, 49528, 71498] avg=65536.0000 max=99656 max_rank=2 min=39248 min_rank=1 imbalance_ratio=1.520630 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] avg=65536.0000 max=66193 max_rank=2 min=63667 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33463 max_load_reduction_pct=0.335785 moved_to_replicas=135239 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[47035, 39248, 99656, 57229, 63828, 96266, 49528, 71498] avg=65536.0000 max=99656 max_rank=2 min=39248 min_rank=1 imbalance_ratio=1.520630 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] avg=65536.0000 max=66193 max_rank=2 min=63667 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33463 max_load_reduction_pct=0.335785 moved_to_replicas=135239 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] avg=65536.0000 max=66193 max_rank=2 min=63667 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33463 max_load_reduction_pct=0.335785 moved_to_replicas=135239 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[47035, 39248, 99656, 57229, 63828, 96266, 49528, 71498] avg=65536.0000 max=99656 max_rank=2 min=39248 min_rank=1 imbalance_ratio=1.520630 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] avg=65536.0000 max=66193 max_rank=2 min=63667 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33463 max_load_reduction_pct=0.335785 moved_to_replicas=135239 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] avg=65536.0000 max=66193 max_rank=2 min=63667 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33463 max_load_reduction_pct=0.335785 moved_to_replicas=135239 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] avg=65536.0000 max=66193 max_rank=2 min=63667 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33463 max_load_reduction_pct=0.335785 moved_to_replicas=135239 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32658, 162244, 113435, 62895, 32575, 21570, 63931, 34980] avg=65536.0000 max=162244 max_rank=1 min=21570 min_rank=5 imbalance_ratio=2.475647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32658, 162244, 113435, 62895, 32575, 21570, 63931, 34980] avg=65536.0000 max=162244 max_rank=1 min=21570 min_rank=5 imbalance_ratio=2.475647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[65831, 66192, 66193, 63667, 63828, 66191, 66193, 66193] avg=65536.0000 max=66193 max_rank=2 min=63667 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33463 max_load_reduction_pct=0.335785 moved_to_replicas=135239 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32658, 162244, 113435, 62895, 32575, 21570, 63931, 34980] avg=65536.0000 max=162244 max_rank=1 min=21570 min_rank=5 imbalance_ratio=2.475647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32658, 162244, 113435, 62895, 32575, 21570, 63931, 34980] avg=65536.0000 max=162244 max_rank=1 min=21570 min_rank=5 imbalance_ratio=2.475647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32658, 162244, 113435, 62895, 32575, 21570, 63931, 34980] avg=65536.0000 max=162244 max_rank=1 min=21570 min_rank=5 imbalance_ratio=2.475647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[32658, 162244, 113435, 62895, 32575, 21570, 63931, 34980] avg=65536.0000 max=162244 max_rank=1 min=21570 min_rank=5 imbalance_ratio=2.475647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] avg=65536.0000 max=66192 max_rank=1 min=63931 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96052 max_load_reduction_pct=0.592022 moved_to_replicas=295692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] avg=65536.0000 max=66192 max_rank=1 min=63931 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96052 max_load_reduction_pct=0.592022 moved_to_replicas=295692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[32658, 162244, 113435, 62895, 32575, 21570, 63931, 34980] avg=65536.0000 max=162244 max_rank=1 min=21570 min_rank=5 imbalance_ratio=2.475647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] avg=65536.0000 max=66192 max_rank=1 min=63931 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96052 max_load_reduction_pct=0.592022 moved_to_replicas=295692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] avg=65536.0000 max=66192 max_rank=1 min=63931 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96052 max_load_reduction_pct=0.592022 moved_to_replicas=295692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32658, 162244, 113435, 62895, 32575, 21570, 63931, 34980] avg=65536.0000 max=162244 max_rank=1 min=21570 min_rank=5 imbalance_ratio=2.475647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] avg=65536.0000 max=66192 max_rank=1 min=63931 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96052 max_load_reduction_pct=0.592022 moved_to_replicas=295692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] avg=65536.0000 max=66192 max_rank=1 min=63931 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96052 max_load_reduction_pct=0.592022 moved_to_replicas=295692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] avg=65536.0000 max=66192 max_rank=1 min=63931 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96052 max_load_reduction_pct=0.592022 moved_to_replicas=295692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[1356, 100493, 65560, 66244, 72876, 61567, 67813, 88379] avg=65536.0000 max=100493 max_rank=1 min=1356 min_rank=0 imbalance_ratio=1.533401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[1356, 100493, 65560, 66244, 72876, 61567, 67813, 88379] avg=65536.0000 max=100493 max_rank=1 min=1356 min_rank=0 imbalance_ratio=1.533401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[1356, 100493, 65560, 66244, 72876, 61567, 67813, 88379] avg=65536.0000 max=100493 max_rank=1 min=1356 min_rank=0 imbalance_ratio=1.533401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[64630, 66192, 66192, 66192, 64767, 66192, 63931, 66192] avg=65536.0000 max=66192 max_rank=1 min=63931 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96052 max_load_reduction_pct=0.592022 moved_to_replicas=295692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[1356, 100493, 65560, 66244, 72876, 61567, 67813, 88379] avg=65536.0000 max=100493 max_rank=1 min=1356 min_rank=0 imbalance_ratio=1.533401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[1356, 100493, 65560, 66244, 72876, 61567, 67813, 88379] avg=65536.0000 max=100493 max_rank=1 min=1356 min_rank=0 imbalance_ratio=1.533401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[1356, 100493, 65560, 66244, 72876, 61567, 67813, 88379] avg=65536.0000 max=100493 max_rank=1 min=1356 min_rank=0 imbalance_ratio=1.533401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63240 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34301 max_load_reduction_pct=0.341327 moved_to_replicas=142200 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63240 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34301 max_load_reduction_pct=0.341327 moved_to_replicas=142200 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63240 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34301 max_load_reduction_pct=0.341327 moved_to_replicas=142200 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63240 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34301 max_load_reduction_pct=0.341327 moved_to_replicas=142200 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[1356, 100493, 65560, 66244, 72876, 61567, 67813, 88379] avg=65536.0000 max=100493 max_rank=1 min=1356 min_rank=0 imbalance_ratio=1.533401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63240 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34301 max_load_reduction_pct=0.341327 moved_to_replicas=142200 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[1356, 100493, 65560, 66244, 72876, 61567, 67813, 88379] avg=65536.0000 max=100493 max_rank=1 min=1356 min_rank=0 imbalance_ratio=1.533401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63240 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34301 max_load_reduction_pct=0.341327 moved_to_replicas=142200 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63240 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34301 max_load_reduction_pct=0.341327 moved_to_replicas=142200 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[96490, 65893, 98028, 65494, 56158, 63591, 13109, 65525] avg=65536.0000 max=98028 max_rank=2 min=13109 min_rank=6 imbalance_ratio=1.495789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[96490, 65893, 98028, 65494, 56158, 63591, 13109, 65525] avg=65536.0000 max=98028 max_rank=2 min=13109 min_rank=6 imbalance_ratio=1.495789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[96490, 65893, 98028, 65494, 56158, 63591, 13109, 65525] avg=65536.0000 max=98028 max_rank=2 min=13109 min_rank=6 imbalance_ratio=1.495789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[96490, 65893, 98028, 65494, 56158, 63591, 13109, 65525] avg=65536.0000 max=98028 max_rank=2 min=13109 min_rank=6 imbalance_ratio=1.495789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[96490, 65893, 98028, 65494, 56158, 63591, 13109, 65525] avg=65536.0000 max=98028 max_rank=2 min=13109 min_rank=6 imbalance_ratio=1.495789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[64529, 66192, 65560, 66192, 66191, 63240, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63240 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34301 max_load_reduction_pct=0.341327 moved_to_replicas=142200 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31836 max_load_reduction_pct=0.324764 moved_to_replicas=128363 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31836 max_load_reduction_pct=0.324764 moved_to_replicas=128363 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[96490, 65893, 98028, 65494, 56158, 63591, 13109, 65525] avg=65536.0000 max=98028 max_rank=2 min=13109 min_rank=6 imbalance_ratio=1.495789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[96490, 65893, 98028, 65494, 56158, 63591, 13109, 65525] avg=65536.0000 max=98028 max_rank=2 min=13109 min_rank=6 imbalance_ratio=1.495789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31836 max_load_reduction_pct=0.324764 moved_to_replicas=128363 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31836 max_load_reduction_pct=0.324764 moved_to_replicas=128363 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31836 max_load_reduction_pct=0.324764 moved_to_replicas=128363 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31836 max_load_reduction_pct=0.324764 moved_to_replicas=128363 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[96490, 65893, 98028, 65494, 56158, 63591, 13109, 65525] avg=65536.0000 max=98028 max_rank=2 min=13109 min_rank=6 imbalance_ratio=1.495789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31836 max_load_reduction_pct=0.324764 moved_to_replicas=128363 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[55181, 65872, 97009, 32829, 131894, 20077, 1610, 119816] avg=65536.0000 max=131894 max_rank=4 min=1610 min_rank=6 imbalance_ratio=2.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[55181, 65872, 97009, 32829, 131894, 20077, 1610, 119816] avg=65536.0000 max=131894 max_rank=4 min=1610 min_rank=6 imbalance_ratio=2.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[55181, 65872, 97009, 32829, 131894, 20077, 1610, 119816] avg=65536.0000 max=131894 max_rank=4 min=1610 min_rank=6 imbalance_ratio=2.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[55181, 65872, 97009, 32829, 131894, 20077, 1610, 119816] avg=65536.0000 max=131894 max_rank=4 min=1610 min_rank=6 imbalance_ratio=2.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[55181, 65872, 97009, 32829, 131894, 20077, 1610, 119816] avg=65536.0000 max=131894 max_rank=4 min=1610 min_rank=6 imbalance_ratio=2.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65893, 66192, 65494, 65209, 63591, 66192, 65525] avg=65536.0000 max=66192 max_rank=0 min=63591 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31836 max_load_reduction_pct=0.324764 moved_to_replicas=128363 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] avg=65536.0000 max=66193 max_rank=5 min=61719 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65701 max_load_reduction_pct=0.498135 moved_to_replicas=299516 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[55181, 65872, 97009, 32829, 131894, 20077, 1610, 119816] avg=65536.0000 max=131894 max_rank=4 min=1610 min_rank=6 imbalance_ratio=2.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] avg=65536.0000 max=66193 max_rank=5 min=61719 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65701 max_load_reduction_pct=0.498135 moved_to_replicas=299516 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] avg=65536.0000 max=66193 max_rank=5 min=61719 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65701 max_load_reduction_pct=0.498135 moved_to_replicas=299516 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[55181, 65872, 97009, 32829, 131894, 20077, 1610, 119816] avg=65536.0000 max=131894 max_rank=4 min=1610 min_rank=6 imbalance_ratio=2.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] avg=65536.0000 max=66193 max_rank=5 min=61719 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65701 max_load_reduction_pct=0.498135 moved_to_replicas=299516 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] avg=65536.0000 max=66193 max_rank=5 min=61719 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65701 max_load_reduction_pct=0.498135 moved_to_replicas=299516 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] avg=65536.0000 max=66193 max_rank=5 min=61719 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65701 max_load_reduction_pct=0.498135 moved_to_replicas=299516 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[55181, 65872, 97009, 32829, 131894, 20077, 1610, 119816] avg=65536.0000 max=131894 max_rank=4 min=1610 min_rank=6 imbalance_ratio=2.012543 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] avg=65536.0000 max=66193 max_rank=5 min=61719 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65701 max_load_reduction_pct=0.498135 moved_to_replicas=299516 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[35377, 108079, 33119, 127658, 26886, 65763, 97571, 29835] avg=65536.0000 max=127658 max_rank=3 min=26886 min_rank=4 imbalance_ratio=1.947906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[35377, 108079, 33119, 127658, 26886, 65763, 97571, 29835] avg=65536.0000 max=127658 max_rank=3 min=26886 min_rank=4 imbalance_ratio=1.947906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[35377, 108079, 33119, 127658, 26886, 65763, 97571, 29835] avg=65536.0000 max=127658 max_rank=3 min=26886 min_rank=4 imbalance_ratio=1.947906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[35377, 108079, 33119, 127658, 26886, 65763, 97571, 29835] avg=65536.0000 max=127658 max_rank=3 min=26886 min_rank=4 imbalance_ratio=1.947906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[35377, 108079, 33119, 127658, 26886, 65763, 97571, 29835] avg=65536.0000 max=127658 max_rank=3 min=26886 min_rank=4 imbalance_ratio=1.947906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65872, 66192, 65738, 66192, 66193, 61719, 66191] avg=65536.0000 max=66193 max_rank=5 min=61719 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65701 max_load_reduction_pct=0.498135 moved_to_replicas=299516 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=61709 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61465 max_load_reduction_pct=0.481482 moved_to_replicas=272569 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=61709 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61465 max_load_reduction_pct=0.481482 moved_to_replicas=272569 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[35377, 108079, 33119, 127658, 26886, 65763, 97571, 29835] avg=65536.0000 max=127658 max_rank=3 min=26886 min_rank=4 imbalance_ratio=1.947906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[35377, 108079, 33119, 127658, 26886, 65763, 97571, 29835] avg=65536.0000 max=127658 max_rank=3 min=26886 min_rank=4 imbalance_ratio=1.947906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=61709 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61465 max_load_reduction_pct=0.481482 moved_to_replicas=272569 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=61709 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61465 max_load_reduction_pct=0.481482 moved_to_replicas=272569 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=61709 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61465 max_load_reduction_pct=0.481482 moved_to_replicas=272569 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=61709 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61465 max_load_reduction_pct=0.481482 moved_to_replicas=272569 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[35377, 108079, 33119, 127658, 26886, 65763, 97571, 29835] avg=65536.0000 max=127658 max_rank=3 min=26886 min_rank=4 imbalance_ratio=1.947906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=61709 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61465 max_load_reduction_pct=0.481482 moved_to_replicas=272569 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[98683, 151, 65601, 55072, 97981, 129080, 45570, 32150] avg=65536.0000 max=129080 max_rank=5 min=151 min_rank=1 imbalance_ratio=1.969604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[98683, 151, 65601, 55072, 97981, 129080, 45570, 32150] avg=65536.0000 max=129080 max_rank=5 min=151 min_rank=1 imbalance_ratio=1.969604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[98683, 151, 65601, 55072, 97981, 129080, 45570, 32150] avg=65536.0000 max=129080 max_rank=5 min=151 min_rank=1 imbalance_ratio=1.969604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[98683, 151, 65601, 55072, 97981, 129080, 45570, 32150] avg=65536.0000 max=129080 max_rank=5 min=151 min_rank=1 imbalance_ratio=1.969604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[98683, 151, 65601, 55072, 97981, 129080, 45570, 32150] avg=65536.0000 max=129080 max_rank=5 min=151 min_rank=1 imbalance_ratio=1.969604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[66191, 66193, 65856, 66192, 61709, 65763, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=61709 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61465 max_load_reduction_pct=0.481482 moved_to_replicas=272569 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[98683, 151, 65601, 55072, 97981, 129080, 45570, 32150] avg=65536.0000 max=129080 max_rank=5 min=151 min_rank=1 imbalance_ratio=1.969604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] avg=65536.0000 max=66192 max_rank=0 min=62350 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62888 max_load_reduction_pct=0.487202 moved_to_replicas=252406 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] avg=65536.0000 max=66192 max_rank=0 min=62350 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62888 max_load_reduction_pct=0.487202 moved_to_replicas=252406 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[98683, 151, 65601, 55072, 97981, 129080, 45570, 32150] avg=65536.0000 max=129080 max_rank=5 min=151 min_rank=1 imbalance_ratio=1.969604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] avg=65536.0000 max=66192 max_rank=0 min=62350 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62888 max_load_reduction_pct=0.487202 moved_to_replicas=252406 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] avg=65536.0000 max=66192 max_rank=0 min=62350 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62888 max_load_reduction_pct=0.487202 moved_to_replicas=252406 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] avg=65536.0000 max=66192 max_rank=0 min=62350 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62888 max_load_reduction_pct=0.487202 moved_to_replicas=252406 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] avg=65536.0000 max=66192 max_rank=0 min=62350 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62888 max_load_reduction_pct=0.487202 moved_to_replicas=252406 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] avg=65536.0000 max=66192 max_rank=0 min=62350 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62888 max_load_reduction_pct=0.487202 moved_to_replicas=252406 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[98683, 151, 65601, 55072, 97981, 129080, 45570, 32150] avg=65536.0000 max=129080 max_rank=5 min=151 min_rank=1 imbalance_ratio=1.969604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[72023, 79840, 67593, 33008, 69600, 102708, 65607, 33909] avg=65536.0000 max=102708 max_rank=5 min=33008 min_rank=3 imbalance_ratio=1.567200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[72023, 79840, 67593, 33008, 69600, 102708, 65607, 33909] avg=65536.0000 max=102708 max_rank=5 min=33008 min_rank=3 imbalance_ratio=1.567200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[72023, 79840, 67593, 33008, 69600, 102708, 65607, 33909] avg=65536.0000 max=102708 max_rank=5 min=33008 min_rank=3 imbalance_ratio=1.567200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[72023, 79840, 67593, 33008, 69600, 102708, 65607, 33909] avg=65536.0000 max=102708 max_rank=5 min=33008 min_rank=3 imbalance_ratio=1.567200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[72023, 79840, 67593, 33008, 69600, 102708, 65607, 33909] avg=65536.0000 max=102708 max_rank=5 min=33008 min_rank=3 imbalance_ratio=1.567200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[72023, 79840, 67593, 33008, 69600, 102708, 65607, 33909] avg=65536.0000 max=102708 max_rank=5 min=33008 min_rank=3 imbalance_ratio=1.567200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65377, 65601, 66192, 66192, 66192, 66192, 62350] avg=65536.0000 max=66192 max_rank=0 min=62350 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62888 max_load_reduction_pct=0.487202 moved_to_replicas=252406 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] avg=65536.0000 max=66192 max_rank=2 min=61988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36516 max_load_reduction_pct=0.355532 moved_to_replicas=127583 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] avg=65536.0000 max=66192 max_rank=2 min=61988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36516 max_load_reduction_pct=0.355532 moved_to_replicas=127583 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[72023, 79840, 67593, 33008, 69600, 102708, 65607, 33909] avg=65536.0000 max=102708 max_rank=5 min=33008 min_rank=3 imbalance_ratio=1.567200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] avg=65536.0000 max=66192 max_rank=2 min=61988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36516 max_load_reduction_pct=0.355532 moved_to_replicas=127583 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] avg=65536.0000 max=66192 max_rank=2 min=61988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36516 max_load_reduction_pct=0.355532 moved_to_replicas=127583 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] avg=65536.0000 max=66192 max_rank=2 min=61988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36516 max_load_reduction_pct=0.355532 moved_to_replicas=127583 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] avg=65536.0000 max=66192 max_rank=2 min=61988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36516 max_load_reduction_pct=0.355532 moved_to_replicas=127583 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] avg=65536.0000 max=66192 max_rank=2 min=61988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36516 max_load_reduction_pct=0.355532 moved_to_replicas=127583 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[72023, 79840, 67593, 33008, 69600, 102708, 65607, 33909] avg=65536.0000 max=102708 max_rank=5 min=33008 min_rank=3 imbalance_ratio=1.567200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[56696, 47145, 61149, 78425, 50780, 35265, 65526, 129302] avg=65536.0000 max=129302 max_rank=7 min=35265 min_rank=5 imbalance_ratio=1.972992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[56696, 47145, 61149, 78425, 50780, 35265, 65526, 129302] avg=65536.0000 max=129302 max_rank=7 min=35265 min_rank=5 imbalance_ratio=1.972992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[56696, 47145, 61149, 78425, 50780, 35265, 65526, 129302] avg=65536.0000 max=129302 max_rank=7 min=35265 min_rank=5 imbalance_ratio=1.972992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[56696, 47145, 61149, 78425, 50780, 35265, 65526, 129302] avg=65536.0000 max=129302 max_rank=7 min=35265 min_rank=5 imbalance_ratio=1.972992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[56696, 47145, 61149, 78425, 50780, 35265, 65526, 129302] avg=65536.0000 max=129302 max_rank=7 min=35265 min_rank=5 imbalance_ratio=1.972992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[56696, 47145, 61149, 78425, 50780, 35265, 65526, 129302] avg=65536.0000 max=129302 max_rank=7 min=35265 min_rank=5 imbalance_ratio=1.972992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] avg=65536.0000 max=66192 max_rank=0 min=63886 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63110 max_load_reduction_pct=0.488082 moved_to_replicas=147323 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[66191, 66191, 66192, 65735, 66192, 66192, 65607, 61988] avg=65536.0000 max=66192 max_rank=2 min=61988 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36516 max_load_reduction_pct=0.355532 moved_to_replicas=127583 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] avg=65536.0000 max=66192 max_rank=0 min=63886 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63110 max_load_reduction_pct=0.488082 moved_to_replicas=147323 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[56696, 47145, 61149, 78425, 50780, 35265, 65526, 129302] avg=65536.0000 max=129302 max_rank=7 min=35265 min_rank=5 imbalance_ratio=1.972992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] avg=65536.0000 max=66192 max_rank=0 min=63886 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63110 max_load_reduction_pct=0.488082 moved_to_replicas=147323 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] avg=65536.0000 max=66192 max_rank=0 min=63886 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63110 max_load_reduction_pct=0.488082 moved_to_replicas=147323 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] avg=65536.0000 max=66192 max_rank=0 min=63886 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63110 max_load_reduction_pct=0.488082 moved_to_replicas=147323 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] avg=65536.0000 max=66192 max_rank=0 min=63886 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63110 max_load_reduction_pct=0.488082 moved_to_replicas=147323 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] avg=65536.0000 max=66192 max_rank=0 min=63886 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63110 max_load_reduction_pct=0.488082 moved_to_replicas=147323 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[98666, 68427, 32833, 2184, 33426, 93680, 162029, 33043] avg=65536.0000 max=162029 max_rank=6 min=2184 min_rank=3 imbalance_ratio=2.472366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[56696, 47145, 61149, 78425, 50780, 35265, 65526, 129302] avg=65536.0000 max=129302 max_rank=7 min=35265 min_rank=5 imbalance_ratio=1.972992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[98666, 68427, 32833, 2184, 33426, 93680, 162029, 33043] avg=65536.0000 max=162029 max_rank=6 min=2184 min_rank=3 imbalance_ratio=2.472366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[98666, 68427, 32833, 2184, 33426, 93680, 162029, 33043] avg=65536.0000 max=162029 max_rank=6 min=2184 min_rank=3 imbalance_ratio=2.472366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[98666, 68427, 32833, 2184, 33426, 93680, 162029, 33043] avg=65536.0000 max=162029 max_rank=6 min=2184 min_rank=3 imbalance_ratio=2.472366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[98666, 68427, 32833, 2184, 33426, 93680, 162029, 33043] avg=65536.0000 max=162029 max_rank=6 min=2184 min_rank=3 imbalance_ratio=2.472366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[98666, 68427, 32833, 2184, 33426, 93680, 162029, 33043] avg=65536.0000 max=162029 max_rank=6 min=2184 min_rank=3 imbalance_ratio=2.472366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] avg=65536.0000 max=66192 max_rank=0 min=63462 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=95837 max_load_reduction_pct=0.591481 moved_to_replicas=307608 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66192, 63886, 66192, 63916, 66192, 65526, 66192] avg=65536.0000 max=66192 max_rank=0 min=63886 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63110 max_load_reduction_pct=0.488082 moved_to_replicas=147323 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] avg=65536.0000 max=66192 max_rank=0 min=63462 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=95837 max_load_reduction_pct=0.591481 moved_to_replicas=307608 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[98666, 68427, 32833, 2184, 33426, 93680, 162029, 33043] avg=65536.0000 max=162029 max_rank=6 min=2184 min_rank=3 imbalance_ratio=2.472366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] avg=65536.0000 max=66192 max_rank=0 min=63462 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=95837 max_load_reduction_pct=0.591481 moved_to_replicas=307608 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] avg=65536.0000 max=66192 max_rank=0 min=63462 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=95837 max_load_reduction_pct=0.591481 moved_to_replicas=307608 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] avg=65536.0000 max=66192 max_rank=0 min=63462 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=95837 max_load_reduction_pct=0.591481 moved_to_replicas=307608 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] avg=65536.0000 max=66192 max_rank=0 min=63462 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=95837 max_load_reduction_pct=0.591481 moved_to_replicas=307608 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] avg=65536.0000 max=66192 max_rank=0 min=63462 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=95837 max_load_reduction_pct=0.591481 moved_to_replicas=307608 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[63371, 100666, 61504, 31764, 63799, 40924, 128511, 33749] avg=65536.0000 max=128511 max_rank=6 min=31764 min_rank=3 imbalance_ratio=1.960922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[98666, 68427, 32833, 2184, 33426, 93680, 162029, 33043] avg=65536.0000 max=162029 max_rank=6 min=2184 min_rank=3 imbalance_ratio=2.472366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[63371, 100666, 61504, 31764, 63799, 40924, 128511, 33749] avg=65536.0000 max=128511 max_rank=6 min=31764 min_rank=3 imbalance_ratio=1.960922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[63371, 100666, 61504, 31764, 63799, 40924, 128511, 33749] avg=65536.0000 max=128511 max_rank=6 min=31764 min_rank=3 imbalance_ratio=1.960922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] avg=65536.0000 max=66194 max_rank=5 min=63799 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=62317 max_load_reduction_pct=0.484916 moved_to_replicas=194536 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[63371, 100666, 61504, 31764, 63799, 40924, 128511, 33749] avg=65536.0000 max=128511 max_rank=6 min=31764 min_rank=3 imbalance_ratio=1.960922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[63371, 100666, 61504, 31764, 63799, 40924, 128511, 33749] avg=65536.0000 max=128511 max_rank=6 min=31764 min_rank=3 imbalance_ratio=1.960922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[63371, 100666, 61504, 31764, 63799, 40924, 128511, 33749] avg=65536.0000 max=128511 max_rank=6 min=31764 min_rank=3 imbalance_ratio=1.960922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] avg=65536.0000 max=66194 max_rank=5 min=63799 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=62317 max_load_reduction_pct=0.484916 moved_to_replicas=194536 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65535, 64625, 65900, 66190, 66192, 63462] avg=65536.0000 max=66192 max_rank=0 min=63462 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=95837 max_load_reduction_pct=0.591481 moved_to_replicas=307608 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] avg=65536.0000 max=66194 max_rank=5 min=63799 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=62317 max_load_reduction_pct=0.484916 moved_to_replicas=194536 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[63371, 100666, 61504, 31764, 63799, 40924, 128511, 33749] avg=65536.0000 max=128511 max_rank=6 min=31764 min_rank=3 imbalance_ratio=1.960922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] avg=65536.0000 max=66194 max_rank=5 min=63799 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=62317 max_load_reduction_pct=0.484916 moved_to_replicas=194536 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] avg=65536.0000 max=66194 max_rank=5 min=63799 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=62317 max_load_reduction_pct=0.484916 moved_to_replicas=194536 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] avg=65536.0000 max=66194 max_rank=5 min=63799 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=62317 max_load_reduction_pct=0.484916 moved_to_replicas=194536 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[44840, 54691, 33038, 65423, 69158, 97418, 97127, 62593] avg=65536.0000 max=97418 max_rank=5 min=33038 min_rank=2 imbalance_ratio=1.486481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] avg=65536.0000 max=66194 max_rank=5 min=63799 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=62317 max_load_reduction_pct=0.484916 moved_to_replicas=194536 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[44840, 54691, 33038, 65423, 69158, 97418, 97127, 62593] avg=65536.0000 max=97418 max_rank=5 min=33038 min_rank=2 imbalance_ratio=1.486481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[44840, 54691, 33038, 65423, 69158, 97418, 97127, 62593] avg=65536.0000 max=97418 max_rank=5 min=33038 min_rank=2 imbalance_ratio=1.486481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[63371, 100666, 61504, 31764, 63799, 40924, 128511, 33749] avg=65536.0000 max=128511 max_rank=6 min=31764 min_rank=3 imbalance_ratio=1.960922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] avg=65536.0000 max=66194 max_rank=0 min=64264 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31224 max_load_reduction_pct=0.320516 moved_to_replicas=123546 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[44840, 54691, 33038, 65423, 69158, 97418, 97127, 62593] avg=65536.0000 max=97418 max_rank=5 min=33038 min_rank=2 imbalance_ratio=1.486481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[44840, 54691, 33038, 65423, 69158, 97418, 97127, 62593] avg=65536.0000 max=97418 max_rank=5 min=33038 min_rank=2 imbalance_ratio=1.486481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] avg=65536.0000 max=66194 max_rank=0 min=64264 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31224 max_load_reduction_pct=0.320516 moved_to_replicas=123546 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[44840, 54691, 33038, 65423, 69158, 97418, 97127, 62593] avg=65536.0000 max=97418 max_rank=5 min=33038 min_rank=2 imbalance_ratio=1.486481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] avg=65536.0000 max=66194 max_rank=0 min=64264 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31224 max_load_reduction_pct=0.320516 moved_to_replicas=123546 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[44840, 54691, 33038, 65423, 69158, 97418, 97127, 62593] avg=65536.0000 max=97418 max_rank=5 min=33038 min_rank=2 imbalance_ratio=1.486481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66190, 64502, 63799, 66194, 66192, 66095] avg=65536.0000 max=66194 max_rank=5 min=63799 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=62317 max_load_reduction_pct=0.484916 moved_to_replicas=194536 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] avg=65536.0000 max=66194 max_rank=0 min=64264 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31224 max_load_reduction_pct=0.320516 moved_to_replicas=123546 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] avg=65536.0000 max=66194 max_rank=0 min=64264 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31224 max_load_reduction_pct=0.320516 moved_to_replicas=123546 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[102111, 30835, 93287, 97934, 91770, 42228, 33755, 32368] avg=65536.0000 max=102111 max_rank=0 min=30835 min_rank=1 imbalance_ratio=1.558090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] avg=65536.0000 max=66194 max_rank=0 min=64264 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31224 max_load_reduction_pct=0.320516 moved_to_replicas=123546 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] avg=65536.0000 max=66194 max_rank=0 min=64264 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31224 max_load_reduction_pct=0.320516 moved_to_replicas=123546 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[102111, 30835, 93287, 97934, 91770, 42228, 33755, 32368] avg=65536.0000 max=102111 max_rank=0 min=30835 min_rank=1 imbalance_ratio=1.558090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[102111, 30835, 93287, 97934, 91770, 42228, 33755, 32368] avg=65536.0000 max=102111 max_rank=0 min=30835 min_rank=1 imbalance_ratio=1.558090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[44840, 54691, 33038, 65423, 69158, 97418, 97127, 62593] avg=65536.0000 max=97418 max_rank=5 min=33038 min_rank=2 imbalance_ratio=1.486481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] avg=65536.0000 max=66193 max_rank=2 min=63466 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35918 max_load_reduction_pct=0.351754 moved_to_replicas=236616 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[102111, 30835, 93287, 97934, 91770, 42228, 33755, 32368] avg=65536.0000 max=102111 max_rank=0 min=30835 min_rank=1 imbalance_ratio=1.558090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] avg=65536.0000 max=66193 max_rank=2 min=63466 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35918 max_load_reduction_pct=0.351754 moved_to_replicas=236616 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[102111, 30835, 93287, 97934, 91770, 42228, 33755, 32368] avg=65536.0000 max=102111 max_rank=0 min=30835 min_rank=1 imbalance_ratio=1.558090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] avg=65536.0000 max=66193 max_rank=2 min=63466 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35918 max_load_reduction_pct=0.351754 moved_to_replicas=236616 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[102111, 30835, 93287, 97934, 91770, 42228, 33755, 32368] avg=65536.0000 max=102111 max_rank=0 min=30835 min_rank=1 imbalance_ratio=1.558090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[66194, 64272, 64264, 65423, 66192, 66192, 66192, 65559] avg=65536.0000 max=66194 max_rank=0 min=64264 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=31224 max_load_reduction_pct=0.320516 moved_to_replicas=123546 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[102111, 30835, 93287, 97934, 91770, 42228, 33755, 32368] avg=65536.0000 max=102111 max_rank=0 min=30835 min_rank=1 imbalance_ratio=1.558090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] avg=65536.0000 max=66193 max_rank=2 min=63466 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35918 max_load_reduction_pct=0.351754 moved_to_replicas=236616 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[65571, 131630, 12776, 43056, 32632, 80841, 63075, 94707] avg=65536.0000 max=131630 max_rank=1 min=12776 min_rank=2 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] avg=65536.0000 max=66193 max_rank=2 min=63466 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35918 max_load_reduction_pct=0.351754 moved_to_replicas=236616 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] avg=65536.0000 max=66193 max_rank=2 min=63466 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35918 max_load_reduction_pct=0.351754 moved_to_replicas=236616 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[65571, 131630, 12776, 43056, 32632, 80841, 63075, 94707] avg=65536.0000 max=131630 max_rank=1 min=12776 min_rank=2 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] avg=65536.0000 max=66193 max_rank=2 min=63466 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35918 max_load_reduction_pct=0.351754 moved_to_replicas=236616 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[65571, 131630, 12776, 43056, 32632, 80841, 63075, 94707] avg=65536.0000 max=131630 max_rank=1 min=12776 min_rank=2 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[102111, 30835, 93287, 97934, 91770, 42228, 33755, 32368] avg=65536.0000 max=102111 max_rank=0 min=30835 min_rank=1 imbalance_ratio=1.558090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] avg=65536.0000 max=66193 max_rank=5 min=63075 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65437 max_load_reduction_pct=0.497128 moved_to_replicas=205601 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[65571, 131630, 12776, 43056, 32632, 80841, 63075, 94707] avg=65536.0000 max=131630 max_rank=1 min=12776 min_rank=2 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] avg=65536.0000 max=66193 max_rank=5 min=63075 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65437 max_load_reduction_pct=0.497128 moved_to_replicas=205601 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] avg=65536.0000 max=66193 max_rank=5 min=63075 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65437 max_load_reduction_pct=0.497128 moved_to_replicas=205601 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[65571, 131630, 12776, 43056, 32632, 80841, 63075, 94707] avg=65536.0000 max=131630 max_rank=1 min=12776 min_rank=2 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[65571, 131630, 12776, 43056, 32632, 80841, 63075, 94707] avg=65536.0000 max=131630 max_rank=1 min=12776 min_rank=2 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[65571, 131630, 12776, 43056, 32632, 80841, 63075, 94707] avg=65536.0000 max=131630 max_rank=1 min=12776 min_rank=2 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63466, 66193, 66192, 66192, 66192, 65497, 64364] avg=65536.0000 max=66193 max_rank=2 min=63466 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35918 max_load_reduction_pct=0.351754 moved_to_replicas=236616 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] avg=65536.0000 max=66193 max_rank=5 min=63075 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65437 max_load_reduction_pct=0.497128 moved_to_replicas=205601 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32869, 52941, 50643, 66596, 131048, 32877, 33281, 124033] avg=65536.0000 max=131048 max_rank=4 min=32869 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32869, 52941, 50643, 66596, 131048, 32877, 33281, 124033] avg=65536.0000 max=131048 max_rank=4 min=32869 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] avg=65536.0000 max=66193 max_rank=5 min=63075 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65437 max_load_reduction_pct=0.497128 moved_to_replicas=205601 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] avg=65536.0000 max=66193 max_rank=5 min=63075 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65437 max_load_reduction_pct=0.497128 moved_to_replicas=205601 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32869, 52941, 50643, 66596, 131048, 32877, 33281, 124033] avg=65536.0000 max=131048 max_rank=4 min=32869 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] avg=65536.0000 max=66193 max_rank=5 min=63075 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65437 max_load_reduction_pct=0.497128 moved_to_replicas=205601 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] avg=65536.0000 max=66193 max_rank=2 min=63805 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64855 max_load_reduction_pct=0.494895 moved_to_replicas=239026 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] avg=65536.0000 max=66193 max_rank=2 min=63805 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64855 max_load_reduction_pct=0.494895 moved_to_replicas=239026 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32869, 52941, 50643, 66596, 131048, 32877, 33281, 124033] avg=65536.0000 max=131048 max_rank=4 min=32869 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[65571, 131630, 12776, 43056, 32632, 80841, 63075, 94707] avg=65536.0000 max=131630 max_rank=1 min=12776 min_rank=2 imbalance_ratio=2.008514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] avg=65536.0000 max=66193 max_rank=2 min=63805 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64855 max_load_reduction_pct=0.494895 moved_to_replicas=239026 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32869, 52941, 50643, 66596, 131048, 32877, 33281, 124033] avg=65536.0000 max=131048 max_rank=4 min=32869 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32869, 52941, 50643, 66596, 131048, 32877, 33281, 124033] avg=65536.0000 max=131048 max_rank=4 min=32869 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32869, 52941, 50643, 66596, 131048, 32877, 33281, 124033] avg=65536.0000 max=131048 max_rank=4 min=32869 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] avg=65536.0000 max=66193 max_rank=2 min=63805 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64855 max_load_reduction_pct=0.494895 moved_to_replicas=239026 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[59578, 80965, 61003, 47314, 96124, 57336, 57865, 64103] avg=65536.0000 max=96124 max_rank=4 min=47314 min_rank=3 imbalance_ratio=1.466736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[65571, 66192, 65576, 66191, 65297, 66193, 63075, 66193] avg=65536.0000 max=66193 max_rank=5 min=63075 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=65437 max_load_reduction_pct=0.497128 moved_to_replicas=205601 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[59578, 80965, 61003, 47314, 96124, 57336, 57865, 64103] avg=65536.0000 max=96124 max_rank=4 min=47314 min_rank=3 imbalance_ratio=1.466736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] avg=65536.0000 max=66193 max_rank=2 min=63805 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64855 max_load_reduction_pct=0.494895 moved_to_replicas=239026 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] avg=65536.0000 max=66193 max_rank=2 min=63805 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64855 max_load_reduction_pct=0.494895 moved_to_replicas=239026 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] avg=65536.0000 max=66193 max_rank=2 min=63805 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64855 max_load_reduction_pct=0.494895 moved_to_replicas=239026 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[59578, 80965, 61003, 47314, 96124, 57336, 57865, 64103] avg=65536.0000 max=96124 max_rank=4 min=47314 min_rank=3 imbalance_ratio=1.466736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] avg=65536.0000 max=66193 max_rank=0 min=63033 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29931 max_load_reduction_pct=0.311379 moved_to_replicas=87479 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] avg=65536.0000 max=66193 max_rank=0 min=63033 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29931 max_load_reduction_pct=0.311379 moved_to_replicas=87479 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[59578, 80965, 61003, 47314, 96124, 57336, 57865, 64103] avg=65536.0000 max=96124 max_rank=4 min=47314 min_rank=3 imbalance_ratio=1.466736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32869, 52941, 50643, 66596, 131048, 32877, 33281, 124033] avg=65536.0000 max=131048 max_rank=4 min=32869 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] avg=65536.0000 max=66193 max_rank=0 min=63033 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29931 max_load_reduction_pct=0.311379 moved_to_replicas=87479 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[59578, 80965, 61003, 47314, 96124, 57336, 57865, 64103] avg=65536.0000 max=96124 max_rank=4 min=47314 min_rank=3 imbalance_ratio=1.466736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[59578, 80965, 61003, 47314, 96124, 57336, 57865, 64103] avg=65536.0000 max=96124 max_rank=4 min=47314 min_rank=3 imbalance_ratio=1.466736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[59578, 80965, 61003, 47314, 96124, 57336, 57865, 64103] avg=65536.0000 max=96124 max_rank=4 min=47314 min_rank=3 imbalance_ratio=1.466736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] avg=65536.0000 max=66193 max_rank=0 min=63033 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29931 max_load_reduction_pct=0.311379 moved_to_replicas=87479 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[65597, 63805, 66193, 66192, 66192, 65005, 65113, 66191] avg=65536.0000 max=66193 max_rank=2 min=63805 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64855 max_load_reduction_pct=0.494895 moved_to_replicas=239026 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] avg=65536.0000 max=66193 max_rank=0 min=63033 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29931 max_load_reduction_pct=0.311379 moved_to_replicas=87479 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] avg=65536.0000 max=66193 max_rank=0 min=63033 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29931 max_load_reduction_pct=0.311379 moved_to_replicas=87479 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] avg=65536.0000 max=66193 max_rank=0 min=63033 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29931 max_load_reduction_pct=0.311379 moved_to_replicas=87479 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[59578, 80965, 61003, 47314, 96124, 57336, 57865, 64103] avg=65536.0000 max=96124 max_rank=4 min=47314 min_rank=3 imbalance_ratio=1.466736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[66193, 66190, 63033, 66192, 66192, 66192, 66193, 64103] avg=65536.0000 max=66193 max_rank=0 min=63033 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29931 max_load_reduction_pct=0.311379 moved_to_replicas=87479 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=2098546 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=106880 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=2103656 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=103296 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=2079957 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=125952 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=2104854 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=100992 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=2096886 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=108544 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=2095569 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=110080 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=2103537 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=101632 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=2094211 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=111872 peak_demand=66194 required_capacity=1.010040 + [2026-08-30 05:15:24] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 542.0 | throughput per GPU (TFLOP/s/GPU): 615.1 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.316014E+01 | loss scale: 1.0 | grad norm: 38.066 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=69632 expected_recv=128095 expected_overflow=0 exact_recv_per_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=69632 expected_recv=128857 expected_overflow=0 exact_recv_per_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=69632 expected_recv=129703 expected_overflow=0 exact_recv_per_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=69632 expected_recv=128095 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=69632 expected_recv=128857 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=69632 expected_recv=129703 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=129536 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=69632 expected_recv=132214 expected_overflow=0 exact_recv_per_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=69632 expected_recv=130713 expected_overflow=0 exact_recv_per_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=69632 expected_recv=132214 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=69632 expected_recv=130713 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=69632 expected_recv=129975 expected_overflow=0 exact_recv_per_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=69632 expected_recv=127411 expected_overflow=0 exact_recv_per_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=69632 expected_recv=131111 expected_overflow=0 exact_recv_per_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=69632 expected_recv=129975 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=69632 expected_recv=127411 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=69632 expected_recv=131111 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=134144 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=129152 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=69632 expected_recv=128542 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=69632 expected_recv=131151 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=69632 expected_recv=127987 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=69632 expected_recv=131360 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=69632 expected_recv=128542 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=69632 expected_recv=131151 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=69632 expected_recv=127987 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=69632 expected_recv=131145 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=69632 expected_recv=131360 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=129664 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=69632 expected_recv=131145 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=69632 expected_recv=131859 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=69632 expected_recv=131024 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=69632 expected_recv=130905 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=69632 expected_recv=131859 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=69632 expected_recv=131393 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=69632 expected_recv=129890 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=69632 expected_recv=131024 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=69632 expected_recv=130905 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=69632 expected_recv=131393 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=69632 expected_recv=129890 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=69632 expected_recv=129976 expected_overflow=0 exact_recv_per_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=69632 expected_recv=129669 expected_overflow=0 exact_recv_per_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=69632 expected_recv=130547 expected_overflow=0 exact_recv_per_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=69632 expected_recv=129976 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=69632 expected_recv=129669 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=69632 expected_recv=130547 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=69632 expected_recv=129827 expected_overflow=0 exact_recv_per_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=69632 expected_recv=130932 expected_overflow=0 exact_recv_per_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=69632 expected_recv=129827 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=69632 expected_recv=130932 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=69632 expected_recv=130449 expected_overflow=0 exact_recv_per_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=69632 expected_recv=128470 expected_overflow=0 exact_recv_per_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=69632 expected_recv=131066 expected_overflow=0 exact_recv_per_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=69632 expected_recv=130449 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=69632 expected_recv=128470 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=69632 expected_recv=131066 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=69632 expected_recv=131190 expected_overflow=0 exact_recv_per_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=69632 expected_recv=131190 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=69632 expected_recv=129956 expected_overflow=0 exact_recv_per_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=69632 expected_recv=129956 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=69632 expected_recv=131566 expected_overflow=0 exact_recv_per_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=69632 expected_recv=131561 expected_overflow=0 exact_recv_per_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=69632 expected_recv=129927 expected_overflow=0 exact_recv_per_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=69632 expected_recv=129920 expected_overflow=0 exact_recv_per_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=69632 expected_recv=131566 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=69632 expected_recv=131561 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=69632 expected_recv=129927 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=69632 expected_recv=129920 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=69632 expected_recv=132233 expected_overflow=0 exact_recv_per_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=69632 expected_recv=128299 expected_overflow=0 exact_recv_per_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=69632 expected_recv=132233 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=69632 expected_recv=128299 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=129664 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=69632 expected_recv=131041 expected_overflow=0 exact_recv_per_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=69632 expected_recv=131142 expected_overflow=0 exact_recv_per_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=69632 expected_recv=131041 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=69632 expected_recv=131142 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=69632 expected_recv=129715 expected_overflow=0 exact_recv_per_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=69632 expected_recv=129776 expected_overflow=0 exact_recv_per_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=69632 expected_recv=130871 expected_overflow=0 exact_recv_per_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=69632 expected_recv=129715 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=69632 expected_recv=129776 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=69632 expected_recv=130871 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=69632 expected_recv=129920 expected_overflow=0 exact_recv_per_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=69632 expected_recv=132074 expected_overflow=0 exact_recv_per_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=69632 expected_recv=129920 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=69632 expected_recv=132074 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=69632 expected_recv=129733 expected_overflow=0 exact_recv_per_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=69632 expected_recv=129718 expected_overflow=0 exact_recv_per_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=69632 expected_recv=129733 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=69632 expected_recv=129718 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=69632 expected_recv=129048 expected_overflow=0 exact_recv_per_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=69632 expected_recv=130849 expected_overflow=0 exact_recv_per_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=69632 expected_recv=129048 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=69632 expected_recv=132033 expected_overflow=0 exact_recv_per_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=69632 expected_recv=130849 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=69632 expected_recv=131180 expected_overflow=0 exact_recv_per_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=69632 expected_recv=132033 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=69632 expected_recv=131356 expected_overflow=0 exact_recv_per_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=69632 expected_recv=130573 expected_overflow=0 exact_recv_per_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=69632 expected_recv=131180 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=69632 expected_recv=131356 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=69632 expected_recv=130573 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=69632 expected_recv=126281 expected_overflow=0 exact_recv_per_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=69632 expected_recv=130383 expected_overflow=0 exact_recv_per_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=69632 expected_recv=126281 actual_valid_recv=127744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=69632 expected_recv=129794 expected_overflow=0 exact_recv_per_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=69632 expected_recv=131733 expected_overflow=0 exact_recv_per_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=127744 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=69632 expected_recv=130383 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=69632 expected_recv=129167 expected_overflow=0 exact_recv_per_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=69632 expected_recv=129794 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=69632 expected_recv=131733 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=69632 expected_recv=129167 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=69632 expected_recv=130971 expected_overflow=0 exact_recv_per_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=69632 expected_recv=130971 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=69632 expected_recv=131761 expected_overflow=0 exact_recv_per_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=69632 expected_recv=131761 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=69632 expected_recv=123658 expected_overflow=0 exact_recv_per_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=69632 expected_recv=123658 actual_valid_recv=125184 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=125184 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=69632 expected_recv=130615 expected_overflow=0 exact_recv_per_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=69632 expected_recv=131423 expected_overflow=0 exact_recv_per_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=69632 expected_recv=130615 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=69632 expected_recv=131663 expected_overflow=0 exact_recv_per_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=69632 expected_recv=131423 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=69632 expected_recv=131663 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=69632 expected_recv=131081 expected_overflow=0 exact_recv_per_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=69632 expected_recv=131081 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=69632 expected_recv=124870 expected_overflow=0 exact_recv_per_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=69632 expected_recv=129693 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=69632 expected_recv=124870 actual_valid_recv=125952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=69632 expected_recv=129693 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=69632 expected_recv=129893 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=125952 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=69632 expected_recv=129893 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=69632 expected_recv=129747 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=69632 expected_recv=129747 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=69632 expected_recv=130572 expected_overflow=0 exact_recv_per_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=69632 expected_recv=130572 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=69632 expected_recv=129706 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=69632 expected_recv=127396 expected_overflow=0 exact_recv_per_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=69632 expected_recv=129706 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=69632 expected_recv=131058 expected_overflow=0 exact_recv_per_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=69632 expected_recv=131127 expected_overflow=0 exact_recv_per_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=69632 expected_recv=127396 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=69632 expected_recv=131058 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=69632 expected_recv=131127 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=128512 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=69632 expected_recv=131267 expected_overflow=0 exact_recv_per_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=69632 expected_recv=129287 expected_overflow=0 exact_recv_per_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=69632 expected_recv=128466 expected_overflow=0 exact_recv_per_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=69632 expected_recv=130767 expected_overflow=0 exact_recv_per_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=69632 expected_recv=131267 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=69632 expected_recv=131804 expected_overflow=0 exact_recv_per_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=69632 expected_recv=129287 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=69632 expected_recv=128466 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=69632 expected_recv=130767 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=69632 expected_recv=131804 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=129152 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[33344, 51836, 101857, 52094, 74249, 85447, 44244, 81217] avg=65536.0000 max=101857 max_rank=2 min=33344 min_rank=0 imbalance_ratio=1.554214 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=69632 expected_recv=131099 expected_overflow=0 exact_recv_per_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[33344, 51836, 101857, 52094, 74249, 85447, 44244, 81217] avg=65536.0000 max=101857 max_rank=2 min=33344 min_rank=0 imbalance_ratio=1.554214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=63511 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35664 max_load_reduction_pct=0.350138 moved_to_replicas=158898 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=69632 expected_recv=131099 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[33344, 51836, 101857, 52094, 74249, 85447, 44244, 81217] avg=65536.0000 max=101857 max_rank=2 min=33344 min_rank=0 imbalance_ratio=1.554214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[33344, 51836, 101857, 52094, 74249, 85447, 44244, 81217] avg=65536.0000 max=101857 max_rank=2 min=33344 min_rank=0 imbalance_ratio=1.554214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[33344, 51836, 101857, 52094, 74249, 85447, 44244, 81217] avg=65536.0000 max=101857 max_rank=2 min=33344 min_rank=0 imbalance_ratio=1.554214 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[33344, 51836, 101857, 52094, 74249, 85447, 44244, 81217] avg=65536.0000 max=101857 max_rank=2 min=33344 min_rank=0 imbalance_ratio=1.554214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[33344, 51836, 101857, 52094, 74249, 85447, 44244, 81217] avg=65536.0000 max=101857 max_rank=2 min=33344 min_rank=0 imbalance_ratio=1.554214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=63511 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35664 max_load_reduction_pct=0.350138 moved_to_replicas=158898 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=63511 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35664 max_load_reduction_pct=0.350138 moved_to_replicas=158898 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=63511 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35664 max_load_reduction_pct=0.350138 moved_to_replicas=158898 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33086, 131049, 66022, 35703, 97298, 65480, 32707, 62943] avg=65536.0000 max=131049 max_rank=1 min=32707 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=63511 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35664 max_load_reduction_pct=0.350138 moved_to_replicas=158898 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=63511 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35664 max_load_reduction_pct=0.350138 moved_to_replicas=158898 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=63511 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35664 max_load_reduction_pct=0.350138 moved_to_replicas=158898 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[33344, 51836, 101857, 52094, 74249, 85447, 44244, 81217] avg=65536.0000 max=101857 max_rank=2 min=33344 min_rank=0 imbalance_ratio=1.554214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33086, 131049, 66022, 35703, 97298, 65480, 32707, 62943] avg=65536.0000 max=131049 max_rank=1 min=32707 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] avg=65536.0000 max=66192 max_rank=1 min=63560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=192099 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33086, 131049, 66022, 35703, 97298, 65480, 32707, 62943] avg=65536.0000 max=131049 max_rank=1 min=32707 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33086, 131049, 66022, 35703, 97298, 65480, 32707, 62943] avg=65536.0000 max=131049 max_rank=1 min=32707 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33086, 131049, 66022, 35703, 97298, 65480, 32707, 62943] avg=65536.0000 max=131049 max_rank=1 min=32707 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[63624, 63511, 66192, 66192, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=4 min=63511 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35664 max_load_reduction_pct=0.350138 moved_to_replicas=158898 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33086, 131049, 66022, 35703, 97298, 65480, 32707, 62943] avg=65536.0000 max=131049 max_rank=1 min=32707 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33086, 131049, 66022, 35703, 97298, 65480, 32707, 62943] avg=65536.0000 max=131049 max_rank=1 min=32707 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] avg=65536.0000 max=66192 max_rank=1 min=63560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=192099 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] avg=65536.0000 max=66192 max_rank=1 min=63560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=192099 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] avg=65536.0000 max=66192 max_rank=1 min=63560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=192099 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[95771, 98213, 90304, 34777, 65808, 6598, 67191, 65626] avg=65536.0000 max=98213 max_rank=1 min=6598 min_rank=5 imbalance_ratio=1.498611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] avg=65536.0000 max=66192 max_rank=1 min=63560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=192099 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] avg=65536.0000 max=66192 max_rank=1 min=63560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=192099 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] avg=65536.0000 max=66192 max_rank=1 min=63560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=192099 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33086, 131049, 66022, 35703, 97298, 65480, 32707, 62943] avg=65536.0000 max=131049 max_rank=1 min=32707 min_rank=6 imbalance_ratio=1.999649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[95771, 98213, 90304, 34777, 65808, 6598, 67191, 65626] avg=65536.0000 max=98213 max_rank=1 min=6598 min_rank=5 imbalance_ratio=1.498611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] avg=65536.0000 max=66192 max_rank=0 min=63730 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32021 max_load_reduction_pct=0.326036 moved_to_replicas=170987 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[95771, 98213, 90304, 34777, 65808, 6598, 67191, 65626] avg=65536.0000 max=98213 max_rank=1 min=6598 min_rank=5 imbalance_ratio=1.498611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[95771, 98213, 90304, 34777, 65808, 6598, 67191, 65626] avg=65536.0000 max=98213 max_rank=1 min=6598 min_rank=5 imbalance_ratio=1.498611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[95771, 98213, 90304, 34777, 65808, 6598, 67191, 65626] avg=65536.0000 max=98213 max_rank=1 min=6598 min_rank=5 imbalance_ratio=1.498611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[95771, 98213, 90304, 34777, 65808, 6598, 67191, 65626] avg=65536.0000 max=98213 max_rank=1 min=6598 min_rank=5 imbalance_ratio=1.498611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] avg=65536.0000 max=66192 max_rank=0 min=63730 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32021 max_load_reduction_pct=0.326036 moved_to_replicas=170987 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[65200, 66192, 66022, 66192, 66192, 65480, 65450, 63560] avg=65536.0000 max=66192 max_rank=1 min=63560 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64857 max_load_reduction_pct=0.494906 moved_to_replicas=192099 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[95771, 98213, 90304, 34777, 65808, 6598, 67191, 65626] avg=65536.0000 max=98213 max_rank=1 min=6598 min_rank=5 imbalance_ratio=1.498611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[98182, 65502, 65622, 65464, 64378, 32913, 66645, 65582] avg=65536.0000 max=98182 max_rank=0 min=32913 min_rank=5 imbalance_ratio=1.498138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] avg=65536.0000 max=66192 max_rank=0 min=63730 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32021 max_load_reduction_pct=0.326036 moved_to_replicas=170987 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] avg=65536.0000 max=66192 max_rank=0 min=63730 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32021 max_load_reduction_pct=0.326036 moved_to_replicas=170987 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] avg=65536.0000 max=66192 max_rank=0 min=63730 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32021 max_load_reduction_pct=0.326036 moved_to_replicas=170987 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] avg=65536.0000 max=66192 max_rank=0 min=63730 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32021 max_load_reduction_pct=0.326036 moved_to_replicas=170987 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] avg=65536.0000 max=66192 max_rank=0 min=63730 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32021 max_load_reduction_pct=0.326036 moved_to_replicas=170987 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[98182, 65502, 65622, 65464, 64378, 32913, 66645, 65582] avg=65536.0000 max=98182 max_rank=0 min=32913 min_rank=5 imbalance_ratio=1.498138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[95771, 98213, 90304, 34777, 65808, 6598, 67191, 65626] avg=65536.0000 max=98213 max_rank=1 min=6598 min_rank=5 imbalance_ratio=1.498611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] avg=65536.0000 max=66192 max_rank=0 min=64831 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.325823 moved_to_replicas=64457 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[98182, 65502, 65622, 65464, 64378, 32913, 66645, 65582] avg=65536.0000 max=98182 max_rank=0 min=32913 min_rank=5 imbalance_ratio=1.498138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[98182, 65502, 65622, 65464, 64378, 32913, 66645, 65582] avg=65536.0000 max=98182 max_rank=0 min=32913 min_rank=5 imbalance_ratio=1.498138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[98182, 65502, 65622, 65464, 64378, 32913, 66645, 65582] avg=65536.0000 max=98182 max_rank=0 min=32913 min_rank=5 imbalance_ratio=1.498138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[98182, 65502, 65622, 65464, 64378, 32913, 66645, 65582] avg=65536.0000 max=98182 max_rank=0 min=32913 min_rank=5 imbalance_ratio=1.498138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] avg=65536.0000 max=66192 max_rank=0 min=64831 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.325823 moved_to_replicas=64457 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66192, 64356, 65808, 63730, 66192, 65626] avg=65536.0000 max=66192 max_rank=0 min=63730 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32021 max_load_reduction_pct=0.326036 moved_to_replicas=170987 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[98182, 65502, 65622, 65464, 64378, 32913, 66645, 65582] avg=65536.0000 max=98182 max_rank=0 min=32913 min_rank=5 imbalance_ratio=1.498138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] avg=65536.0000 max=66192 max_rank=0 min=64831 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.325823 moved_to_replicas=64457 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32859, 98016, 64630, 65455, 98936, 32992, 33133, 98267] avg=65536.0000 max=98936 max_rank=4 min=32859 min_rank=0 imbalance_ratio=1.509644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] avg=65536.0000 max=66192 max_rank=0 min=64831 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.325823 moved_to_replicas=64457 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] avg=65536.0000 max=66192 max_rank=0 min=64831 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.325823 moved_to_replicas=64457 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] avg=65536.0000 max=66192 max_rank=0 min=64831 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.325823 moved_to_replicas=64457 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] avg=65536.0000 max=66192 max_rank=0 min=64831 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.325823 moved_to_replicas=64457 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32859, 98016, 64630, 65455, 98936, 32992, 33133, 98267] avg=65536.0000 max=98936 max_rank=4 min=32859 min_rank=0 imbalance_ratio=1.509644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] avg=65536.0000 max=66192 max_rank=1 min=64630 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32744 max_load_reduction_pct=0.330961 moved_to_replicas=193320 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[98182, 65502, 65622, 65464, 64378, 32913, 66645, 65582] avg=65536.0000 max=98182 max_rank=0 min=32913 min_rank=5 imbalance_ratio=1.498138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32859, 98016, 64630, 65455, 98936, 32992, 33133, 98267] avg=65536.0000 max=98936 max_rank=4 min=32859 min_rank=0 imbalance_ratio=1.509644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32859, 98016, 64630, 65455, 98936, 32992, 33133, 98267] avg=65536.0000 max=98936 max_rank=4 min=32859 min_rank=0 imbalance_ratio=1.509644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32859, 98016, 64630, 65455, 98936, 32992, 33133, 98267] avg=65536.0000 max=98936 max_rank=4 min=32859 min_rank=0 imbalance_ratio=1.509644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] avg=65536.0000 max=66192 max_rank=1 min=64630 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32744 max_load_reduction_pct=0.330961 moved_to_replicas=193320 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32859, 98016, 64630, 65455, 98936, 32992, 33133, 98267] avg=65536.0000 max=98936 max_rank=4 min=32859 min_rank=0 imbalance_ratio=1.509644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65502, 65622, 65464, 64831, 64903, 66192, 65582] avg=65536.0000 max=66192 max_rank=0 min=64831 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.325823 moved_to_replicas=64457 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] avg=65536.0000 max=66192 max_rank=1 min=64630 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32744 max_load_reduction_pct=0.330961 moved_to_replicas=193320 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32859, 98016, 64630, 65455, 98936, 32992, 33133, 98267] avg=65536.0000 max=98936 max_rank=4 min=32859 min_rank=0 imbalance_ratio=1.509644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32778, 152742, 38818, 65476, 65526, 65860, 38091, 64997] avg=65536.0000 max=152742 max_rank=1 min=32778 min_rank=0 imbalance_ratio=2.330658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] avg=65536.0000 max=66192 max_rank=1 min=64630 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32744 max_load_reduction_pct=0.330961 moved_to_replicas=193320 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] avg=65536.0000 max=66192 max_rank=1 min=64630 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32744 max_load_reduction_pct=0.330961 moved_to_replicas=193320 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] avg=65536.0000 max=66192 max_rank=1 min=64630 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32744 max_load_reduction_pct=0.330961 moved_to_replicas=193320 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32778, 152742, 38818, 65476, 65526, 65860, 38091, 64997] avg=65536.0000 max=152742 max_rank=1 min=32778 min_rank=0 imbalance_ratio=2.330658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] avg=65536.0000 max=66192 max_rank=1 min=64630 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32744 max_load_reduction_pct=0.330961 moved_to_replicas=193320 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] avg=65536.0000 max=66193 max_rank=6 min=64502 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=86549 max_load_reduction_pct=0.566635 moved_to_replicas=168379 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32778, 152742, 38818, 65476, 65526, 65860, 38091, 64997] avg=65536.0000 max=152742 max_rank=1 min=32778 min_rank=0 imbalance_ratio=2.330658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32859, 98016, 64630, 65455, 98936, 32992, 33133, 98267] avg=65536.0000 max=98936 max_rank=4 min=32859 min_rank=0 imbalance_ratio=1.509644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] avg=65536.0000 max=66193 max_rank=6 min=64502 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=86549 max_load_reduction_pct=0.566635 moved_to_replicas=168379 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32778, 152742, 38818, 65476, 65526, 65860, 38091, 64997] avg=65536.0000 max=152742 max_rank=1 min=32778 min_rank=0 imbalance_ratio=2.330658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32778, 152742, 38818, 65476, 65526, 65860, 38091, 64997] avg=65536.0000 max=152742 max_rank=1 min=32778 min_rank=0 imbalance_ratio=2.330658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32778, 152742, 38818, 65476, 65526, 65860, 38091, 64997] avg=65536.0000 max=152742 max_rank=1 min=32778 min_rank=0 imbalance_ratio=2.330658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] avg=65536.0000 max=66193 max_rank=6 min=64502 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=86549 max_load_reduction_pct=0.566635 moved_to_replicas=168379 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[65603, 66192, 64630, 65455, 66192, 65067, 64957, 66192] avg=65536.0000 max=66192 max_rank=1 min=64630 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32744 max_load_reduction_pct=0.330961 moved_to_replicas=193320 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[62681, 27062, 32770, 116902, 65526, 85067, 67041, 67239] avg=65536.0000 max=116902 max_rank=3 min=27062 min_rank=1 imbalance_ratio=1.783783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32778, 152742, 38818, 65476, 65526, 65860, 38091, 64997] avg=65536.0000 max=152742 max_rank=1 min=32778 min_rank=0 imbalance_ratio=2.330658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] avg=65536.0000 max=66193 max_rank=6 min=64502 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=86549 max_load_reduction_pct=0.566635 moved_to_replicas=168379 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] avg=65536.0000 max=66193 max_rank=6 min=64502 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=86549 max_load_reduction_pct=0.566635 moved_to_replicas=168379 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] avg=65536.0000 max=66193 max_rank=6 min=64502 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=86549 max_load_reduction_pct=0.566635 moved_to_replicas=168379 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[62681, 27062, 32770, 116902, 65526, 85067, 67041, 67239] avg=65536.0000 max=116902 max_rank=3 min=27062 min_rank=1 imbalance_ratio=1.783783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=63728 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=50709 max_load_reduction_pct=0.433774 moved_to_replicas=138540 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] avg=65536.0000 max=66193 max_rank=6 min=64502 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=86549 max_load_reduction_pct=0.566635 moved_to_replicas=168379 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[62681, 27062, 32770, 116902, 65526, 85067, 67041, 67239] avg=65536.0000 max=116902 max_rank=3 min=27062 min_rank=1 imbalance_ratio=1.783783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32778, 152742, 38818, 65476, 65526, 65860, 38091, 64997] avg=65536.0000 max=152742 max_rank=1 min=32778 min_rank=0 imbalance_ratio=2.330658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=63728 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=50709 max_load_reduction_pct=0.433774 moved_to_replicas=138540 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[62681, 27062, 32770, 116902, 65526, 85067, 67041, 67239] avg=65536.0000 max=116902 max_rank=3 min=27062 min_rank=1 imbalance_ratio=1.783783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[62681, 27062, 32770, 116902, 65526, 85067, 67041, 67239] avg=65536.0000 max=116902 max_rank=3 min=27062 min_rank=1 imbalance_ratio=1.783783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[62681, 27062, 32770, 116902, 65526, 85067, 67041, 67239] avg=65536.0000 max=116902 max_rank=3 min=27062 min_rank=1 imbalance_ratio=1.783783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=63728 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=50709 max_load_reduction_pct=0.433774 moved_to_replicas=138540 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[65544, 66190, 64502, 65476, 65526, 65860, 66193, 64997] avg=65536.0000 max=66193 max_rank=6 min=64502 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=86549 max_load_reduction_pct=0.566635 moved_to_replicas=168379 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65330, 65582, 32896, 98305, 32771, 32900, 98179] avg=65536.0000 max=98325 max_rank=0 min=32771 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=63728 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=50709 max_load_reduction_pct=0.433774 moved_to_replicas=138540 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[62681, 27062, 32770, 116902, 65526, 85067, 67041, 67239] avg=65536.0000 max=116902 max_rank=3 min=27062 min_rank=1 imbalance_ratio=1.783783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=63728 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=50709 max_load_reduction_pct=0.433774 moved_to_replicas=138540 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=63728 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=50709 max_load_reduction_pct=0.433774 moved_to_replicas=138540 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65330, 65582, 32896, 98305, 32771, 32900, 98179] avg=65536.0000 max=98325 max_rank=0 min=32771 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] avg=65536.0000 max=66192 max_rank=0 min=64887 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191934 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65330, 65582, 32896, 98305, 32771, 32900, 98179] avg=65536.0000 max=98325 max_rank=0 min=32771 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=63728 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=50709 max_load_reduction_pct=0.433774 moved_to_replicas=138540 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[62681, 27062, 32770, 116902, 65526, 85067, 67041, 67239] avg=65536.0000 max=116902 max_rank=3 min=27062 min_rank=1 imbalance_ratio=1.783783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] avg=65536.0000 max=66192 max_rank=0 min=64887 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191934 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65330, 65582, 32896, 98305, 32771, 32900, 98179] avg=65536.0000 max=98325 max_rank=0 min=32771 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65330, 65582, 32896, 98305, 32771, 32900, 98179] avg=65536.0000 max=98325 max_rank=0 min=32771 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65330, 65582, 32896, 98305, 32771, 32900, 98179] avg=65536.0000 max=98325 max_rank=0 min=32771 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] avg=65536.0000 max=66192 max_rank=0 min=64887 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191934 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[63728, 64074, 66190, 66193, 65526, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=63728 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=50709 max_load_reduction_pct=0.433774 moved_to_replicas=138540 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[98228, 32817, 66302, 98274, 35776, 65330, 94770, 32791] avg=65536.0000 max=98274 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] avg=65536.0000 max=66192 max_rank=0 min=64887 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191934 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] avg=65536.0000 max=66192 max_rank=0 min=64887 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191934 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] avg=65536.0000 max=66192 max_rank=0 min=64887 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191934 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65330, 65582, 32896, 98305, 32771, 32900, 98179] avg=65536.0000 max=98325 max_rank=0 min=32771 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[98228, 32817, 66302, 98274, 35776, 65330, 94770, 32791] avg=65536.0000 max=98274 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] avg=65536.0000 max=66194 max_rank=6 min=64462 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32080 max_load_reduction_pct=0.326434 moved_to_replicas=182550 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[98228, 32817, 66302, 98274, 35776, 65330, 94770, 32791] avg=65536.0000 max=98274 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[98228, 32817, 66302, 98274, 35776, 65330, 94770, 32791] avg=65536.0000 max=98274 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65330, 65582, 32896, 98305, 32771, 32900, 98179] avg=65536.0000 max=98325 max_rank=0 min=32771 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] avg=65536.0000 max=66192 max_rank=0 min=64887 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191934 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] avg=65536.0000 max=66194 max_rank=6 min=64462 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32080 max_load_reduction_pct=0.326434 moved_to_replicas=182550 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[98228, 32817, 66302, 98274, 35776, 65330, 94770, 32791] avg=65536.0000 max=98274 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] avg=65536.0000 max=66194 max_rank=6 min=64462 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32080 max_load_reduction_pct=0.326434 moved_to_replicas=182550 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[98228, 32817, 66302, 98274, 35776, 65330, 94770, 32791] avg=65536.0000 max=98274 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] avg=65536.0000 max=66194 max_rank=6 min=64462 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32080 max_load_reduction_pct=0.326434 moved_to_replicas=182550 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 92014, 71817, 97894, 3, 65631, 65413] avg=65536.0000 max=97894 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65330, 65582, 65009, 66192, 64904, 64887, 66192] avg=65536.0000 max=66192 max_rank=0 min=64887 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191934 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] avg=65536.0000 max=66194 max_rank=6 min=64462 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32080 max_load_reduction_pct=0.326434 moved_to_replicas=182550 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] avg=65536.0000 max=66194 max_rank=6 min=64462 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32080 max_load_reduction_pct=0.326434 moved_to_replicas=182550 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[98228, 32817, 66302, 98274, 35776, 65330, 94770, 32791] avg=65536.0000 max=98274 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 92014, 71817, 97894, 3, 65631, 65413] avg=65536.0000 max=97894 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] avg=65536.0000 max=66193 max_rank=2 min=63152 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=31701 max_load_reduction_pct=0.323830 moved_to_replicas=126046 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 92014, 71817, 97894, 3, 65631, 65413] avg=65536.0000 max=97894 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 92014, 71817, 97894, 3, 65631, 65413] avg=65536.0000 max=97894 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[98228, 32817, 66302, 98274, 35776, 65330, 94770, 32791] avg=65536.0000 max=98274 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] avg=65536.0000 max=66194 max_rank=6 min=64462 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32080 max_load_reduction_pct=0.326434 moved_to_replicas=182550 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] avg=65536.0000 max=66193 max_rank=2 min=63152 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=31701 max_load_reduction_pct=0.323830 moved_to_replicas=126046 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 92014, 71817, 97894, 3, 65631, 65413] avg=65536.0000 max=97894 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] avg=65536.0000 max=66193 max_rank=2 min=63152 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=31701 max_load_reduction_pct=0.323830 moved_to_replicas=126046 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 92014, 71817, 97894, 3, 65631, 65413] avg=65536.0000 max=97894 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65272, 98556, 167403, 9, 65532, 32822, 29609, 65085] avg=65536.0000 max=167403 max_rank=2 min=9 min_rank=3 imbalance_ratio=2.554367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] avg=65536.0000 max=66193 max_rank=2 min=63152 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=31701 max_load_reduction_pct=0.323830 moved_to_replicas=126046 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64853, 66192, 66192, 64462, 65330, 66194, 64873] avg=65536.0000 max=66194 max_rank=6 min=64462 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32080 max_load_reduction_pct=0.326434 moved_to_replicas=182550 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] avg=65536.0000 max=66193 max_rank=2 min=63152 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=31701 max_load_reduction_pct=0.323830 moved_to_replicas=126046 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] avg=65536.0000 max=66193 max_rank=2 min=63152 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=31701 max_load_reduction_pct=0.323830 moved_to_replicas=126046 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65272, 98556, 167403, 9, 65532, 32822, 29609, 65085] avg=65536.0000 max=167403 max_rank=2 min=9 min_rank=3 imbalance_ratio=2.554367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 92014, 71817, 97894, 3, 65631, 65413] avg=65536.0000 max=97894 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] avg=65536.0000 max=66192 max_rank=1 min=64284 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=101211 max_load_reduction_pct=0.604595 moved_to_replicas=273392 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65272, 98556, 167403, 9, 65532, 32822, 29609, 65085] avg=65536.0000 max=167403 max_rank=2 min=9 min_rank=3 imbalance_ratio=2.554367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65272, 98556, 167403, 9, 65532, 32822, 29609, 65085] avg=65536.0000 max=167403 max_rank=2 min=9 min_rank=3 imbalance_ratio=2.554367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 92014, 71817, 97894, 3, 65631, 65413] avg=65536.0000 max=97894 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65272, 98556, 167403, 9, 65532, 32822, 29609, 65085] avg=65536.0000 max=167403 max_rank=2 min=9 min_rank=3 imbalance_ratio=2.554367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] avg=65536.0000 max=66192 max_rank=1 min=64284 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=101211 max_load_reduction_pct=0.604595 moved_to_replicas=273392 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] avg=65536.0000 max=66193 max_rank=2 min=63152 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=31701 max_load_reduction_pct=0.323830 moved_to_replicas=126046 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] avg=65536.0000 max=66192 max_rank=1 min=64284 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=101211 max_load_reduction_pct=0.604595 moved_to_replicas=273392 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65272, 98556, 167403, 9, 65532, 32822, 29609, 65085] avg=65536.0000 max=167403 max_rank=2 min=9 min_rank=3 imbalance_ratio=2.554367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] avg=65536.0000 max=66192 max_rank=1 min=64284 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=101211 max_load_reduction_pct=0.604595 moved_to_replicas=273392 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[23484, 39808, 98283, 96941, 98326, 65534, 3754, 98158] avg=65536.0000 max=98326 max_rank=4 min=3754 min_rank=6 imbalance_ratio=1.500336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65841, 65675, 66193, 66191, 66192, 63152, 65631, 65413] avg=65536.0000 max=66193 max_rank=2 min=63152 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=31701 max_load_reduction_pct=0.323830 moved_to_replicas=126046 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] avg=65536.0000 max=66192 max_rank=1 min=64284 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=101211 max_load_reduction_pct=0.604595 moved_to_replicas=273392 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[23484, 39808, 98283, 96941, 98326, 65534, 3754, 98158] avg=65536.0000 max=98326 max_rank=4 min=3754 min_rank=6 imbalance_ratio=1.500336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] avg=65536.0000 max=66192 max_rank=1 min=64284 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=101211 max_load_reduction_pct=0.604595 moved_to_replicas=273392 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65272, 98556, 167403, 9, 65532, 32822, 29609, 65085] avg=65536.0000 max=167403 max_rank=2 min=9 min_rank=3 imbalance_ratio=2.554367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61604 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.326811 moved_to_replicas=250841 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[23484, 39808, 98283, 96941, 98326, 65534, 3754, 98158] avg=65536.0000 max=98326 max_rank=4 min=3754 min_rank=6 imbalance_ratio=1.500336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[23484, 39808, 98283, 96941, 98326, 65534, 3754, 98158] avg=65536.0000 max=98326 max_rank=4 min=3754 min_rank=6 imbalance_ratio=1.500336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65272, 98556, 167403, 9, 65532, 32822, 29609, 65085] avg=65536.0000 max=167403 max_rank=2 min=9 min_rank=3 imbalance_ratio=2.554367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[23484, 39808, 98283, 96941, 98326, 65534, 3754, 98158] avg=65536.0000 max=98326 max_rank=4 min=3754 min_rank=6 imbalance_ratio=1.500336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61604 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.326811 moved_to_replicas=250841 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] avg=65536.0000 max=66192 max_rank=1 min=64284 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=101211 max_load_reduction_pct=0.604595 moved_to_replicas=273392 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61604 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.326811 moved_to_replicas=250841 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61604 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.326811 moved_to_replicas=250841 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[23484, 39808, 98283, 96941, 98326, 65534, 3754, 98158] avg=65536.0000 max=98326 max_rank=4 min=3754 min_rank=6 imbalance_ratio=1.500336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[55851, 108743, 38, 32768, 65574, 62545, 100552, 98217] avg=65536.0000 max=108743 max_rank=1 min=38 min_rank=2 imbalance_ratio=1.659286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65272, 66192, 66192, 65539, 65532, 66192, 64284, 65085] avg=65536.0000 max=66192 max_rank=1 min=64284 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=101211 max_load_reduction_pct=0.604595 moved_to_replicas=273392 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61604 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.326811 moved_to_replicas=250841 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[55851, 108743, 38, 32768, 65574, 62545, 100552, 98217] avg=65536.0000 max=108743 max_rank=1 min=38 min_rank=2 imbalance_ratio=1.659286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62545 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42550 max_load_reduction_pct=0.391290 moved_to_replicas=218542 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61604 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.326811 moved_to_replicas=250841 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[23484, 39808, 98283, 96941, 98326, 65534, 3754, 98158] avg=65536.0000 max=98326 max_rank=4 min=3754 min_rank=6 imbalance_ratio=1.500336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[55851, 108743, 38, 32768, 65574, 62545, 100552, 98217] avg=65536.0000 max=108743 max_rank=1 min=38 min_rank=2 imbalance_ratio=1.659286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[55851, 108743, 38, 32768, 65574, 62545, 100552, 98217] avg=65536.0000 max=108743 max_rank=1 min=38 min_rank=2 imbalance_ratio=1.659286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[23484, 39808, 98283, 96941, 98326, 65534, 3754, 98158] avg=65536.0000 max=98326 max_rank=4 min=3754 min_rank=6 imbalance_ratio=1.500336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[55851, 108743, 38, 32768, 65574, 62545, 100552, 98217] avg=65536.0000 max=108743 max_rank=1 min=38 min_rank=2 imbalance_ratio=1.659286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62545 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42550 max_load_reduction_pct=0.391290 moved_to_replicas=218542 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61604 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.326811 moved_to_replicas=250841 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62545 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42550 max_load_reduction_pct=0.391290 moved_to_replicas=218542 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62545 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42550 max_load_reduction_pct=0.391290 moved_to_replicas=218542 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[25976, 98291, 72945, 64987, 32770, 32740, 98261, 98318] avg=65536.0000 max=98318 max_rank=7 min=25976 min_rank=0 imbalance_ratio=1.500214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[55851, 108743, 38, 32768, 65574, 62545, 100552, 98217] avg=65536.0000 max=108743 max_rank=1 min=38 min_rank=2 imbalance_ratio=1.659286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[61604, 66190, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=61604 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32134 max_load_reduction_pct=0.326811 moved_to_replicas=250841 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62545 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42550 max_load_reduction_pct=0.391290 moved_to_replicas=218542 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[25976, 98291, 72945, 64987, 32770, 32740, 98261, 98318] avg=65536.0000 max=98318 max_rank=7 min=25976 min_rank=0 imbalance_ratio=1.500214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64839 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32126 max_load_reduction_pct=0.326756 moved_to_replicas=209968 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62545 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42550 max_load_reduction_pct=0.391290 moved_to_replicas=218542 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[55851, 108743, 38, 32768, 65574, 62545, 100552, 98217] avg=65536.0000 max=108743 max_rank=1 min=38 min_rank=2 imbalance_ratio=1.659286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[25976, 98291, 72945, 64987, 32770, 32740, 98261, 98318] avg=65536.0000 max=98318 max_rank=7 min=25976 min_rank=0 imbalance_ratio=1.500214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[25976, 98291, 72945, 64987, 32770, 32740, 98261, 98318] avg=65536.0000 max=98318 max_rank=7 min=25976 min_rank=0 imbalance_ratio=1.500214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64839 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32126 max_load_reduction_pct=0.326756 moved_to_replicas=209968 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[25976, 98291, 72945, 64987, 32770, 32740, 98261, 98318] avg=65536.0000 max=98318 max_rank=7 min=25976 min_rank=0 imbalance_ratio=1.500214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[55851, 108743, 38, 32768, 65574, 62545, 100552, 98217] avg=65536.0000 max=108743 max_rank=1 min=38 min_rank=2 imbalance_ratio=1.659286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62545 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42550 max_load_reduction_pct=0.391290 moved_to_replicas=218542 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64839 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32126 max_load_reduction_pct=0.326756 moved_to_replicas=209968 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64839 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32126 max_load_reduction_pct=0.326756 moved_to_replicas=209968 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[63014, 97801, 72454, 93894, 65568, 65581, 32771, 33205] avg=65536.0000 max=97801 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[25976, 98291, 72945, 64987, 32770, 32740, 98261, 98318] avg=65536.0000 max=98318 max_rank=7 min=25976 min_rank=0 imbalance_ratio=1.500214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64839 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32126 max_load_reduction_pct=0.326756 moved_to_replicas=209968 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[65830, 66193, 65570, 66192, 65574, 62545, 66192, 66192] avg=65536.0000 max=66193 max_rank=1 min=62545 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42550 max_load_reduction_pct=0.391290 moved_to_replicas=218542 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[63014, 97801, 72454, 93894, 65568, 65581, 32771, 33205] avg=65536.0000 max=97801 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] avg=65536.0000 max=66194 max_rank=2 min=63991 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31607 max_load_reduction_pct=0.323177 moved_to_replicas=131089 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64839 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32126 max_load_reduction_pct=0.326756 moved_to_replicas=209968 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[63014, 97801, 72454, 93894, 65568, 65581, 32771, 33205] avg=65536.0000 max=97801 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[63014, 97801, 72454, 93894, 65568, 65581, 32771, 33205] avg=65536.0000 max=97801 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[25976, 98291, 72945, 64987, 32770, 32740, 98261, 98318] avg=65536.0000 max=98318 max_rank=7 min=25976 min_rank=0 imbalance_ratio=1.500214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] avg=65536.0000 max=66194 max_rank=2 min=63991 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31607 max_load_reduction_pct=0.323177 moved_to_replicas=131089 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[63014, 97801, 72454, 93894, 65568, 65581, 32771, 33205] avg=65536.0000 max=97801 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[25976, 98291, 72945, 64987, 32770, 32740, 98261, 98318] avg=65536.0000 max=98318 max_rank=7 min=25976 min_rank=0 imbalance_ratio=1.500214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] avg=65536.0000 max=66194 max_rank=2 min=63991 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31607 max_load_reduction_pct=0.323177 moved_to_replicas=131089 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] avg=65536.0000 max=66194 max_rank=2 min=63991 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31607 max_load_reduction_pct=0.323177 moved_to_replicas=131089 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96999, 129803, 65612, 65538, 92430, 41126] avg=65536.0000 max=129803 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.980637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64839 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32126 max_load_reduction_pct=0.326756 moved_to_replicas=209968 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[63014, 97801, 72454, 93894, 65568, 65581, 32771, 33205] avg=65536.0000 max=97801 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] avg=65536.0000 max=66194 max_rank=2 min=63991 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31607 max_load_reduction_pct=0.323177 moved_to_replicas=131089 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[64856, 66192, 66191, 64987, 64839, 64839, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64839 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32126 max_load_reduction_pct=0.326756 moved_to_replicas=209968 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96999, 129803, 65612, 65538, 92430, 41126] avg=65536.0000 max=129803 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.980637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63612 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63611 max_load_reduction_pct=0.490058 moved_to_replicas=242598 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96999, 129803, 65612, 65538, 92430, 41126] avg=65536.0000 max=129803 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.980637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] avg=65536.0000 max=66194 max_rank=2 min=63991 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31607 max_load_reduction_pct=0.323177 moved_to_replicas=131089 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96999, 129803, 65612, 65538, 92430, 41126] avg=65536.0000 max=129803 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.980637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[63014, 97801, 72454, 93894, 65568, 65581, 32771, 33205] avg=65536.0000 max=97801 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63612 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63611 max_load_reduction_pct=0.490058 moved_to_replicas=242598 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96999, 129803, 65612, 65538, 92430, 41126] avg=65536.0000 max=129803 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.980637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63612 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63611 max_load_reduction_pct=0.490058 moved_to_replicas=242598 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[63014, 97801, 72454, 93894, 65568, 65581, 32771, 33205] avg=65536.0000 max=97801 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63612 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63611 max_load_reduction_pct=0.490058 moved_to_replicas=242598 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] avg=65536.0000 max=66194 max_rank=2 min=63991 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31607 max_load_reduction_pct=0.323177 moved_to_replicas=131089 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96999, 129803, 65612, 65538, 92430, 41126] avg=65536.0000 max=129803 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.980637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63612 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63611 max_load_reduction_pct=0.490058 moved_to_replicas=242598 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[63991, 66192, 66194, 66194, 65568, 65581, 64380, 66188] avg=65536.0000 max=66194 max_rank=2 min=63991 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=31607 max_load_reduction_pct=0.323177 moved_to_replicas=131089 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63612 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63611 max_load_reduction_pct=0.490058 moved_to_replicas=242598 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96999, 129803, 65612, 65538, 92430, 41126] avg=65536.0000 max=129803 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.980637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96999, 129803, 65612, 65538, 92430, 41126] avg=65536.0000 max=129803 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.980637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63612 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63611 max_load_reduction_pct=0.490058 moved_to_replicas=242598 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[64758, 63612, 66192, 66192, 65612, 65538, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63612 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=63611 max_load_reduction_pct=0.490058 moved_to_replicas=242598 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=2100765 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=108160 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=2104290 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=104448 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=2097882 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=111616 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=2102026 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=109056 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=2100787 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=108928 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=2105140 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=102784 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=2084190 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=124800 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=2082136 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=127104 peak_demand=66193 required_capacity=1.010025 + [2026-08-30 05:15:25] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 516.7 | throughput per GPU (TFLOP/s/GPU): 645.3 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.038098E+01 | loss scale: 1.0 | grad norm: 16.219 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=69632 expected_recv=129475 expected_overflow=0 exact_recv_per_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=69632 expected_recv=131142 expected_overflow=0 exact_recv_per_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=69632 expected_recv=126035 expected_overflow=0 exact_recv_per_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=69632 expected_recv=129475 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=69632 expected_recv=131142 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=69632 expected_recv=126035 actual_valid_recv=127616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=127616 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=69632 expected_recv=130372 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=69632 expected_recv=128347 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=69632 expected_recv=131039 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=69632 expected_recv=130372 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=69632 expected_recv=128347 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=69632 expected_recv=131013 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=69632 expected_recv=131039 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=69632 expected_recv=130653 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=69632 expected_recv=131013 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=69632 expected_recv=130653 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=69632 expected_recv=128340 expected_overflow=0 exact_recv_per_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=69632 expected_recv=131056 expected_overflow=0 exact_recv_per_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=69632 expected_recv=128340 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=69632 expected_recv=130944 expected_overflow=0 exact_recv_per_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=69632 expected_recv=131056 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=129792 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=69632 expected_recv=130944 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=69632 expected_recv=131052 expected_overflow=0 exact_recv_per_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=69632 expected_recv=130033 expected_overflow=0 exact_recv_per_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=69632 expected_recv=131052 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=69632 expected_recv=130033 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=69632 expected_recv=130694 expected_overflow=0 exact_recv_per_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=69632 expected_recv=130723 expected_overflow=0 exact_recv_per_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=69632 expected_recv=130694 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=69632 expected_recv=130723 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=69632 expected_recv=126615 expected_overflow=0 exact_recv_per_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=134144 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=69632 expected_recv=126615 actual_valid_recv=128256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=69632 expected_recv=131008 expected_overflow=0 exact_recv_per_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=128256 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=69632 expected_recv=131008 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=69632 expected_recv=131034 expected_overflow=0 exact_recv_per_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=69632 expected_recv=131034 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=69632 expected_recv=129040 expected_overflow=0 exact_recv_per_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=69632 expected_recv=130470 expected_overflow=0 exact_recv_per_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=69632 expected_recv=129040 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=69632 expected_recv=131760 expected_overflow=0 exact_recv_per_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=69632 expected_recv=130642 expected_overflow=0 exact_recv_per_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=69632 expected_recv=130470 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=69632 expected_recv=130862 expected_overflow=0 exact_recv_per_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=69632 expected_recv=130642 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=69632 expected_recv=131760 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=69632 expected_recv=130862 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=69632 expected_recv=129901 expected_overflow=0 exact_recv_per_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=69632 expected_recv=129901 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=69632 expected_recv=128166 expected_overflow=0 exact_recv_per_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=69632 expected_recv=131057 expected_overflow=0 exact_recv_per_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=69632 expected_recv=128166 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=69632 expected_recv=131057 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=69632 expected_recv=129913 expected_overflow=0 exact_recv_per_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=69632 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=69632 expected_recv=129913 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=69632 expected_recv=132387 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=134144 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=134272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=134272 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=69632 expected_recv=131013 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=69632 expected_recv=131059 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=69632 expected_recv=125882 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=69632 expected_recv=131013 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=69632 expected_recv=131059 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=69632 expected_recv=131086 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=69632 expected_recv=131436 expected_overflow=0 exact_recv_per_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=69632 expected_recv=125882 actual_valid_recv=127488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=69632 expected_recv=131086 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=69632 expected_recv=131011 expected_overflow=0 exact_recv_per_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=69632 expected_recv=131436 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=127488 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=69632 expected_recv=129646 expected_overflow=0 exact_recv_per_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=69632 expected_recv=131011 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=69632 expected_recv=129646 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=69632 expected_recv=129621 expected_overflow=0 exact_recv_per_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=69632 expected_recv=129621 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=69632 expected_recv=129710 expected_overflow=0 exact_recv_per_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=69632 expected_recv=129710 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=69632 expected_recv=129595 expected_overflow=0 exact_recv_per_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=69632 expected_recv=129595 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=69632 expected_recv=130422 expected_overflow=0 exact_recv_per_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=69632 expected_recv=130422 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=69632 expected_recv=130416 expected_overflow=0 exact_recv_per_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=69632 expected_recv=129785 expected_overflow=0 exact_recv_per_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=69632 expected_recv=130416 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=69632 expected_recv=131206 expected_overflow=0 exact_recv_per_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=69632 expected_recv=129785 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=69632 expected_recv=131150 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=69632 expected_recv=131206 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=69632 expected_recv=131150 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=69632 expected_recv=131137 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=69632 expected_recv=131137 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=69632 expected_recv=128056 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=69632 expected_recv=130030 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=69632 expected_recv=128056 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=69632 expected_recv=128698 expected_overflow=0 exact_recv_per_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=69632 expected_recv=130030 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=129024 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=69632 expected_recv=128698 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=69632 expected_recv=129692 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=69632 expected_recv=129692 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=69632 expected_recv=129641 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=69632 expected_recv=131097 expected_overflow=0 exact_recv_per_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=69632 expected_recv=129677 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=69632 expected_recv=129641 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=69632 expected_recv=131110 expected_overflow=0 exact_recv_per_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=69632 expected_recv=131097 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=69632 expected_recv=129699 expected_overflow=0 exact_recv_per_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=69632 expected_recv=129677 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=69632 expected_recv=131110 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=69632 expected_recv=129699 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=69632 expected_recv=127138 expected_overflow=0 exact_recv_per_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=69632 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=69632 expected_recv=127138 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=129024 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=69632 expected_recv=132381 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=69632 expected_recv=129693 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=69632 expected_recv=129693 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=69632 expected_recv=130002 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=69632 expected_recv=130002 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=69632 expected_recv=130912 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=69632 expected_recv=130912 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=69632 expected_recv=130472 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=69632 expected_recv=130472 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=69632 expected_recv=131049 expected_overflow=0 exact_recv_per_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=69632 expected_recv=131049 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=69632 expected_recv=129714 expected_overflow=0 exact_recv_per_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=69632 expected_recv=130345 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=69632 expected_recv=129714 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=69632 expected_recv=130345 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=69632 expected_recv=131531 expected_overflow=0 exact_recv_per_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=69632 expected_recv=131531 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=69632 expected_recv=131096 expected_overflow=0 exact_recv_per_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=69632 expected_recv=129486 expected_overflow=0 exact_recv_per_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=69632 expected_recv=130634 expected_overflow=0 exact_recv_per_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=69632 expected_recv=131096 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=69632 expected_recv=129644 expected_overflow=0 exact_recv_per_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=69632 expected_recv=129486 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=69632 expected_recv=131068 expected_overflow=0 exact_recv_per_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=69632 expected_recv=130634 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=69632 expected_recv=129644 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=69632 expected_recv=131068 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=69632 expected_recv=128836 expected_overflow=0 exact_recv_per_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=69632 expected_recv=129791 expected_overflow=0 exact_recv_per_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=69632 expected_recv=132388 expected_overflow=0 exact_recv_per_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=69632 expected_recv=128836 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=69632 expected_recv=129791 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=69632 expected_recv=132388 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=69632 expected_recv=130636 expected_overflow=0 exact_recv_per_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=69632 expected_recv=131095 expected_overflow=0 exact_recv_per_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=69632 expected_recv=130636 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=69632 expected_recv=131095 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=69632 expected_recv=127695 expected_overflow=0 exact_recv_per_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=69632 expected_recv=130878 expected_overflow=0 exact_recv_per_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=69632 expected_recv=127695 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[24175, 80353, 87918, 45167, 90912, 68694, 50527, 76542] avg=65536.0000 max=90912 max_rank=4 min=24175 min_rank=0 imbalance_ratio=1.387207 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=69632 expected_recv=130878 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=128768 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[24175, 80353, 87918, 45167, 90912, 68694, 50527, 76542] avg=65536.0000 max=90912 max_rank=4 min=24175 min_rank=0 imbalance_ratio=1.387207 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=69632 expected_recv=131117 expected_overflow=0 exact_recv_per_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[24175, 80353, 87918, 45167, 90912, 68694, 50527, 76542] avg=65536.0000 max=90912 max_rank=4 min=24175 min_rank=0 imbalance_ratio=1.387207 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=69632 expected_recv=131117 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[24175, 80353, 87918, 45167, 90912, 68694, 50527, 76542] avg=65536.0000 max=90912 max_rank=4 min=24175 min_rank=0 imbalance_ratio=1.387207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] avg=65536.0000 max=66193 max_rank=7 min=63378 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24719 max_load_reduction_pct=0.271900 moved_to_replicas=146480 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] avg=65536.0000 max=66193 max_rank=7 min=63378 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24719 max_load_reduction_pct=0.271900 moved_to_replicas=146480 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[24175, 80353, 87918, 45167, 90912, 68694, 50527, 76542] avg=65536.0000 max=90912 max_rank=4 min=24175 min_rank=0 imbalance_ratio=1.387207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[24175, 80353, 87918, 45167, 90912, 68694, 50527, 76542] avg=65536.0000 max=90912 max_rank=4 min=24175 min_rank=0 imbalance_ratio=1.387207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] avg=65536.0000 max=66193 max_rank=7 min=63378 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24719 max_load_reduction_pct=0.271900 moved_to_replicas=146480 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[24175, 80353, 87918, 45167, 90912, 68694, 50527, 76542] avg=65536.0000 max=90912 max_rank=4 min=24175 min_rank=0 imbalance_ratio=1.387207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] avg=65536.0000 max=66193 max_rank=7 min=63378 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24719 max_load_reduction_pct=0.271900 moved_to_replicas=146480 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] avg=65536.0000 max=66193 max_rank=7 min=63378 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24719 max_load_reduction_pct=0.271900 moved_to_replicas=146480 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98443, 32829, 83152, 113173, 65554, 65514, 32838] avg=65536.0000 max=113173 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98443, 32829, 83152, 113173, 65554, 65514, 32838] avg=65536.0000 max=113173 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] avg=65536.0000 max=66193 max_rank=7 min=63378 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24719 max_load_reduction_pct=0.271900 moved_to_replicas=146480 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[24175, 80353, 87918, 45167, 90912, 68694, 50527, 76542] avg=65536.0000 max=90912 max_rank=4 min=24175 min_rank=0 imbalance_ratio=1.387207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] avg=65536.0000 max=66193 max_rank=7 min=63378 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24719 max_load_reduction_pct=0.271900 moved_to_replicas=146480 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98443, 32829, 83152, 113173, 65554, 65514, 32838] avg=65536.0000 max=113173 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] avg=65536.0000 max=66192 max_rank=1 min=64043 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=46981 max_load_reduction_pct=0.415126 moved_to_replicas=192449 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98443, 32829, 83152, 113173, 65554, 65514, 32838] avg=65536.0000 max=113173 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] avg=65536.0000 max=66192 max_rank=1 min=64043 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=46981 max_load_reduction_pct=0.415126 moved_to_replicas=192449 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98443, 32829, 83152, 113173, 65554, 65514, 32838] avg=65536.0000 max=113173 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[63757, 66192, 66192, 66192, 66192, 66192, 63378, 66193] avg=65536.0000 max=66193 max_rank=7 min=63378 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24719 max_load_reduction_pct=0.271900 moved_to_replicas=146480 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] avg=65536.0000 max=66192 max_rank=1 min=64043 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=46981 max_load_reduction_pct=0.415126 moved_to_replicas=192449 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98443, 32829, 83152, 113173, 65554, 65514, 32838] avg=65536.0000 max=113173 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98443, 32829, 83152, 113173, 65554, 65514, 32838] avg=65536.0000 max=113173 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] avg=65536.0000 max=66192 max_rank=1 min=64043 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=46981 max_load_reduction_pct=0.415126 moved_to_replicas=192449 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] avg=65536.0000 max=66192 max_rank=1 min=64043 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=46981 max_load_reduction_pct=0.415126 moved_to_replicas=192449 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[96883, 33136, 32781, 32775, 65139, 117281, 65530, 80763] avg=65536.0000 max=117281 max_rank=5 min=32775 min_rank=3 imbalance_ratio=1.789566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[96883, 33136, 32781, 32775, 65139, 117281, 65530, 80763] avg=65536.0000 max=117281 max_rank=5 min=32775 min_rank=3 imbalance_ratio=1.789566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] avg=65536.0000 max=66192 max_rank=1 min=64043 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=46981 max_load_reduction_pct=0.415126 moved_to_replicas=192449 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] avg=65536.0000 max=66192 max_rank=1 min=64043 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=46981 max_load_reduction_pct=0.415126 moved_to_replicas=192449 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98443, 32829, 83152, 113173, 65554, 65514, 32838] avg=65536.0000 max=113173 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[96883, 33136, 32781, 32775, 65139, 117281, 65530, 80763] avg=65536.0000 max=117281 max_rank=5 min=32775 min_rank=3 imbalance_ratio=1.789566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] avg=65536.0000 max=66192 max_rank=0 min=63827 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=51089 max_load_reduction_pct=0.435612 moved_to_replicas=192605 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] avg=65536.0000 max=66192 max_rank=0 min=63827 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=51089 max_load_reduction_pct=0.435612 moved_to_replicas=192605 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[96883, 33136, 32781, 32775, 65139, 117281, 65530, 80763] avg=65536.0000 max=117281 max_rank=5 min=32775 min_rank=3 imbalance_ratio=1.789566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[96883, 33136, 32781, 32775, 65139, 117281, 65530, 80763] avg=65536.0000 max=117281 max_rank=5 min=32775 min_rank=3 imbalance_ratio=1.789566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64043, 66191, 66192, 65554, 65514, 65089] avg=65536.0000 max=66192 max_rank=1 min=64043 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=46981 max_load_reduction_pct=0.415126 moved_to_replicas=192449 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] avg=65536.0000 max=66192 max_rank=0 min=63827 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=51089 max_load_reduction_pct=0.435612 moved_to_replicas=192605 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[96883, 33136, 32781, 32775, 65139, 117281, 65530, 80763] avg=65536.0000 max=117281 max_rank=5 min=32775 min_rank=3 imbalance_ratio=1.789566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[96883, 33136, 32781, 32775, 65139, 117281, 65530, 80763] avg=65536.0000 max=117281 max_rank=5 min=32775 min_rank=3 imbalance_ratio=1.789566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] avg=65536.0000 max=66192 max_rank=0 min=63827 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=51089 max_load_reduction_pct=0.435612 moved_to_replicas=192605 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] avg=65536.0000 max=66192 max_rank=0 min=63827 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=51089 max_load_reduction_pct=0.435612 moved_to_replicas=192605 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32603, 65400, 62218, 48855, 65496, 95333, 108433, 45950] avg=65536.0000 max=108433 max_rank=6 min=32603 min_rank=0 imbalance_ratio=1.654556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32603, 65400, 62218, 48855, 65496, 95333, 108433, 45950] avg=65536.0000 max=108433 max_rank=6 min=32603 min_rank=0 imbalance_ratio=1.654556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] avg=65536.0000 max=66192 max_rank=0 min=63827 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=51089 max_load_reduction_pct=0.435612 moved_to_replicas=192605 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] avg=65536.0000 max=66192 max_rank=0 min=63827 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=51089 max_load_reduction_pct=0.435612 moved_to_replicas=192605 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32603, 65400, 62218, 48855, 65496, 95333, 108433, 45950] avg=65536.0000 max=108433 max_rank=6 min=32603 min_rank=0 imbalance_ratio=1.654556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[96883, 33136, 32781, 32775, 65139, 117281, 65530, 80763] avg=65536.0000 max=117281 max_rank=5 min=32775 min_rank=3 imbalance_ratio=1.789566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63259 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=42239 max_load_reduction_pct=0.389540 moved_to_replicas=145673 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63259 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=42239 max_load_reduction_pct=0.389540 moved_to_replicas=145673 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32603, 65400, 62218, 48855, 65496, 95333, 108433, 45950] avg=65536.0000 max=108433 max_rank=6 min=32603 min_rank=0 imbalance_ratio=1.654556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32603, 65400, 62218, 48855, 65496, 95333, 108433, 45950] avg=65536.0000 max=108433 max_rank=6 min=32603 min_rank=0 imbalance_ratio=1.654556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63259 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=42239 max_load_reduction_pct=0.389540 moved_to_replicas=145673 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[66192, 63827, 65683, 65534, 65139, 66192, 65530, 66191] avg=65536.0000 max=66192 max_rank=0 min=63827 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=51089 max_load_reduction_pct=0.435612 moved_to_replicas=192605 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32603, 65400, 62218, 48855, 65496, 95333, 108433, 45950] avg=65536.0000 max=108433 max_rank=6 min=32603 min_rank=0 imbalance_ratio=1.654556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32603, 65400, 62218, 48855, 65496, 95333, 108433, 45950] avg=65536.0000 max=108433 max_rank=6 min=32603 min_rank=0 imbalance_ratio=1.654556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 98302, 131046, 33458, 32836, 33056] avg=65536.0000 max=131046 max_rank=4 min=32836 min_rank=6 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63259 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=42239 max_load_reduction_pct=0.389540 moved_to_replicas=145673 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 98302, 131046, 33458, 32836, 33056] avg=65536.0000 max=131046 max_rank=4 min=32836 min_rank=6 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63259 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=42239 max_load_reduction_pct=0.389540 moved_to_replicas=145673 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63259 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=42239 max_load_reduction_pct=0.389540 moved_to_replicas=145673 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63259 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=42239 max_load_reduction_pct=0.389540 moved_to_replicas=145673 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 98302, 131046, 33458, 32836, 33056] avg=65536.0000 max=131046 max_rank=4 min=32836 min_rank=6 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] avg=65536.0000 max=66192 max_rank=3 min=64821 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=193897 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] avg=65536.0000 max=66192 max_rank=3 min=64821 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=193897 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32603, 65400, 62218, 48855, 65496, 95333, 108433, 45950] avg=65536.0000 max=108433 max_rank=6 min=32603 min_rank=0 imbalance_ratio=1.654556 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 98302, 131046, 33458, 32836, 33056] avg=65536.0000 max=131046 max_rank=4 min=32836 min_rank=6 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] avg=65536.0000 max=66192 max_rank=3 min=64821 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=193897 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 98302, 131046, 33458, 32836, 33056] avg=65536.0000 max=131046 max_rank=4 min=32836 min_rank=6 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 98302, 131046, 33458, 32836, 33056] avg=65536.0000 max=131046 max_rank=4 min=32836 min_rank=6 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 98302, 131046, 33458, 32836, 33056] avg=65536.0000 max=131046 max_rank=4 min=32836 min_rank=6 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[65365, 65400, 63259, 66190, 65496, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63259 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=42239 max_load_reduction_pct=0.389540 moved_to_replicas=145673 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[11743, 70630, 65528, 98299, 85400, 127086, 32748, 32854] avg=65536.0000 max=127086 max_rank=5 min=11743 min_rank=0 imbalance_ratio=1.939178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[11743, 70630, 65528, 98299, 85400, 127086, 32748, 32854] avg=65536.0000 max=127086 max_rank=5 min=11743 min_rank=0 imbalance_ratio=1.939178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] avg=65536.0000 max=66192 max_rank=3 min=64821 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=193897 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] avg=65536.0000 max=66192 max_rank=3 min=64821 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=193897 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] avg=65536.0000 max=66192 max_rank=3 min=64821 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=193897 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[11743, 70630, 65528, 98299, 85400, 127086, 32748, 32854] avg=65536.0000 max=127086 max_rank=5 min=11743 min_rank=0 imbalance_ratio=1.939178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] avg=65536.0000 max=66192 max_rank=3 min=64821 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=193897 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] avg=65536.0000 max=66193 max_rank=4 min=63711 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60893 max_load_reduction_pct=0.479148 moved_to_replicas=227487 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] avg=65536.0000 max=66193 max_rank=4 min=63711 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60893 max_load_reduction_pct=0.479148 moved_to_replicas=227487 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 98302, 131046, 33458, 32836, 33056] avg=65536.0000 max=131046 max_rank=4 min=32836 min_rank=6 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[11743, 70630, 65528, 98299, 85400, 127086, 32748, 32854] avg=65536.0000 max=127086 max_rank=5 min=11743 min_rank=0 imbalance_ratio=1.939178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] avg=65536.0000 max=66193 max_rank=4 min=63711 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60893 max_load_reduction_pct=0.479148 moved_to_replicas=227487 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[11743, 70630, 65528, 98299, 85400, 127086, 32748, 32854] avg=65536.0000 max=127086 max_rank=5 min=11743 min_rank=0 imbalance_ratio=1.939178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[11743, 70630, 65528, 98299, 85400, 127086, 32748, 32854] avg=65536.0000 max=127086 max_rank=5 min=11743 min_rank=0 imbalance_ratio=1.939178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[11743, 70630, 65528, 98299, 85400, 127086, 32748, 32854] avg=65536.0000 max=127086 max_rank=5 min=11743 min_rank=0 imbalance_ratio=1.939178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[64821, 65513, 65256, 66192, 66192, 65568, 65596, 65150] avg=65536.0000 max=66192 max_rank=3 min=64821 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=193897 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32788, 25290, 65516, 98287, 32785, 59675, 65529, 144418] avg=65536.0000 max=144418 max_rank=7 min=25290 min_rank=1 imbalance_ratio=2.203644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32788, 25290, 65516, 98287, 32785, 59675, 65529, 144418] avg=65536.0000 max=144418 max_rank=7 min=25290 min_rank=1 imbalance_ratio=2.203644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] avg=65536.0000 max=66193 max_rank=4 min=63711 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60893 max_load_reduction_pct=0.479148 moved_to_replicas=227487 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] avg=65536.0000 max=66193 max_rank=4 min=63711 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60893 max_load_reduction_pct=0.479148 moved_to_replicas=227487 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] avg=65536.0000 max=66193 max_rank=4 min=63711 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60893 max_load_reduction_pct=0.479148 moved_to_replicas=227487 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] avg=65536.0000 max=66193 max_rank=4 min=63711 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60893 max_load_reduction_pct=0.479148 moved_to_replicas=227487 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32788, 25290, 65516, 98287, 32785, 59675, 65529, 144418] avg=65536.0000 max=144418 max_rank=7 min=25290 min_rank=1 imbalance_ratio=2.203644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] avg=65536.0000 max=66193 max_rank=0 min=62934 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=78225 max_load_reduction_pct=0.541657 moved_to_replicas=219420 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] avg=65536.0000 max=66193 max_rank=0 min=62934 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=78225 max_load_reduction_pct=0.541657 moved_to_replicas=219420 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[11743, 70630, 65528, 98299, 85400, 127086, 32748, 32854] avg=65536.0000 max=127086 max_rank=5 min=11743 min_rank=0 imbalance_ratio=1.939178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32788, 25290, 65516, 98287, 32785, 59675, 65529, 144418] avg=65536.0000 max=144418 max_rank=7 min=25290 min_rank=1 imbalance_ratio=2.203644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] avg=65536.0000 max=66193 max_rank=0 min=62934 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=78225 max_load_reduction_pct=0.541657 moved_to_replicas=219420 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32788, 25290, 65516, 98287, 32785, 59675, 65529, 144418] avg=65536.0000 max=144418 max_rank=7 min=25290 min_rank=1 imbalance_ratio=2.203644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32788, 25290, 65516, 98287, 32785, 59675, 65529, 144418] avg=65536.0000 max=144418 max_rank=7 min=25290 min_rank=1 imbalance_ratio=2.203644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32788, 25290, 65516, 98287, 32785, 59675, 65529, 144418] avg=65536.0000 max=144418 max_rank=7 min=25290 min_rank=1 imbalance_ratio=2.203644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[63711, 66192, 65528, 66192, 66193, 66192, 65319, 64961] avg=65536.0000 max=66193 max_rank=4 min=63711 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60893 max_load_reduction_pct=0.479148 moved_to_replicas=227487 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[21551, 32939, 131013, 65526, 131042, 24, 11699, 130494] avg=65536.0000 max=131042 max_rank=4 min=24 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[21551, 32939, 131013, 65526, 131042, 24, 11699, 130494] avg=65536.0000 max=131042 max_rank=4 min=24 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] avg=65536.0000 max=66193 max_rank=0 min=62934 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=78225 max_load_reduction_pct=0.541657 moved_to_replicas=219420 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] avg=65536.0000 max=66193 max_rank=0 min=62934 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=78225 max_load_reduction_pct=0.541657 moved_to_replicas=219420 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] avg=65536.0000 max=66193 max_rank=0 min=62934 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=78225 max_load_reduction_pct=0.541657 moved_to_replicas=219420 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] avg=65536.0000 max=66193 max_rank=0 min=62934 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=78225 max_load_reduction_pct=0.541657 moved_to_replicas=219420 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[21551, 32939, 131013, 65526, 131042, 24, 11699, 130494] avg=65536.0000 max=131042 max_rank=4 min=24 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63454 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64849 max_load_reduction_pct=0.494872 moved_to_replicas=387406 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63454 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64849 max_load_reduction_pct=0.494872 moved_to_replicas=387406 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32788, 25290, 65516, 98287, 32785, 59675, 65529, 144418] avg=65536.0000 max=144418 max_rank=7 min=25290 min_rank=1 imbalance_ratio=2.203644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[21551, 32939, 131013, 65526, 131042, 24, 11699, 130494] avg=65536.0000 max=131042 max_rank=4 min=24 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63454 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64849 max_load_reduction_pct=0.494872 moved_to_replicas=387406 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[21551, 32939, 131013, 65526, 131042, 24, 11699, 130494] avg=65536.0000 max=131042 max_rank=4 min=24 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[21551, 32939, 131013, 65526, 131042, 24, 11699, 130494] avg=65536.0000 max=131042 max_rank=4 min=24 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[21551, 32939, 131013, 65526, 131042, 24, 11699, 130494] avg=65536.0000 max=131042 max_rank=4 min=24 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[66193, 66192, 65516, 66192, 65541, 62934, 65529, 66191] avg=65536.0000 max=66193 max_rank=0 min=62934 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=78225 max_load_reduction_pct=0.541657 moved_to_replicas=219420 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[33274, 97691, 21, 131051, 32832, 65647, 130984, 32788] avg=65536.0000 max=131051 max_rank=3 min=21 min_rank=2 imbalance_ratio=1.999680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[33274, 97691, 21, 131051, 32832, 65647, 130984, 32788] avg=65536.0000 max=131051 max_rank=3 min=21 min_rank=2 imbalance_ratio=1.999680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63454 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64849 max_load_reduction_pct=0.494872 moved_to_replicas=387406 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63454 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64849 max_load_reduction_pct=0.494872 moved_to_replicas=387406 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63454 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64849 max_load_reduction_pct=0.494872 moved_to_replicas=387406 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63454 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64849 max_load_reduction_pct=0.494872 moved_to_replicas=387406 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[33274, 97691, 21, 131051, 32832, 65647, 130984, 32788] avg=65536.0000 max=131051 max_rank=3 min=21 min_rank=2 imbalance_ratio=1.999680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64773 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.494914 moved_to_replicas=321493 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64773 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.494914 moved_to_replicas=321493 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[21551, 32939, 131013, 65526, 131042, 24, 11699, 130494] avg=65536.0000 max=131042 max_rank=4 min=24 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64773 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.494914 moved_to_replicas=321493 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[33274, 97691, 21, 131051, 32832, 65647, 130984, 32788] avg=65536.0000 max=131051 max_rank=3 min=21 min_rank=2 imbalance_ratio=1.999680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[33274, 97691, 21, 131051, 32832, 65647, 130984, 32788] avg=65536.0000 max=131051 max_rank=3 min=21 min_rank=2 imbalance_ratio=1.999680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[33274, 97691, 21, 131051, 32832, 65647, 130984, 32788] avg=65536.0000 max=131051 max_rank=3 min=21 min_rank=2 imbalance_ratio=1.999680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[33274, 97691, 21, 131051, 32832, 65647, 130984, 32788] avg=65536.0000 max=131051 max_rank=3 min=21 min_rank=2 imbalance_ratio=1.999680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[63454, 65697, 66192, 65526, 66192, 64842, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63454 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64849 max_load_reduction_pct=0.494872 moved_to_replicas=387406 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65579, 98290, 65551, 98341, 68571, 28, 95171, 32757] avg=65536.0000 max=98341 max_rank=3 min=28 min_rank=5 imbalance_ratio=1.500565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65579, 98290, 65551, 98341, 68571, 28, 95171, 32757] avg=65536.0000 max=98341 max_rank=3 min=28 min_rank=5 imbalance_ratio=1.500565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64773 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.494914 moved_to_replicas=321493 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64773 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.494914 moved_to_replicas=321493 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64773 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.494914 moved_to_replicas=321493 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65579, 98290, 65551, 98341, 68571, 28, 95171, 32757] avg=65536.0000 max=98341 max_rank=3 min=28 min_rank=5 imbalance_ratio=1.500565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64773 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.494914 moved_to_replicas=321493 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] avg=65536.0000 max=66192 max_rank=1 min=64115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.326913 moved_to_replicas=191176 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] avg=65536.0000 max=66192 max_rank=1 min=64115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.326913 moved_to_replicas=191176 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[33274, 97691, 21, 131051, 32832, 65647, 130984, 32788] avg=65536.0000 max=131051 max_rank=3 min=21 min_rank=2 imbalance_ratio=1.999680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] avg=65536.0000 max=66192 max_rank=1 min=64115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.326913 moved_to_replicas=191176 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65579, 98290, 65551, 98341, 68571, 28, 95171, 32757] avg=65536.0000 max=98341 max_rank=3 min=28 min_rank=5 imbalance_ratio=1.500565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65579, 98290, 65551, 98341, 68571, 28, 95171, 32757] avg=65536.0000 max=98341 max_rank=3 min=28 min_rank=5 imbalance_ratio=1.500565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65579, 98290, 65551, 98341, 68571, 28, 95171, 32757] avg=65536.0000 max=98341 max_rank=3 min=28 min_rank=5 imbalance_ratio=1.500565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65579, 98290, 65551, 98341, 68571, 28, 95171, 32757] avg=65536.0000 max=98341 max_rank=3 min=28 min_rank=5 imbalance_ratio=1.500565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[64773, 66192, 64880, 66192, 64866, 65647, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64773 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64859 max_load_reduction_pct=0.494914 moved_to_replicas=321493 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[98196, 98103, 163920, 32762, 139, 98370, 32756, 42] avg=65536.0000 max=163920 max_rank=2 min=42 min_rank=7 imbalance_ratio=2.501221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[98196, 98103, 163920, 32762, 139, 98370, 32756, 42] avg=65536.0000 max=163920 max_rank=2 min=42 min_rank=7 imbalance_ratio=2.501221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] avg=65536.0000 max=66192 max_rank=1 min=64115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.326913 moved_to_replicas=191176 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] avg=65536.0000 max=66192 max_rank=1 min=64115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.326913 moved_to_replicas=191176 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[98196, 98103, 163920, 32762, 139, 98370, 32756, 42] avg=65536.0000 max=163920 max_rank=2 min=42 min_rank=7 imbalance_ratio=2.501221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] avg=65536.0000 max=66192 max_rank=1 min=64115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.326913 moved_to_replicas=191176 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] avg=65536.0000 max=66192 max_rank=1 min=64115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.326913 moved_to_replicas=191176 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] avg=65536.0000 max=66192 max_rank=0 min=64805 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97728 max_load_reduction_pct=0.596193 moved_to_replicas=387741 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] avg=65536.0000 max=66192 max_rank=0 min=64805 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97728 max_load_reduction_pct=0.596193 moved_to_replicas=387741 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65579, 98290, 65551, 98341, 68571, 28, 95171, 32757] avg=65536.0000 max=98341 max_rank=3 min=28 min_rank=5 imbalance_ratio=1.500565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] avg=65536.0000 max=66192 max_rank=0 min=64805 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97728 max_load_reduction_pct=0.596193 moved_to_replicas=387741 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[98196, 98103, 163920, 32762, 139, 98370, 32756, 42] avg=65536.0000 max=163920 max_rank=2 min=42 min_rank=7 imbalance_ratio=2.501221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[98196, 98103, 163920, 32762, 139, 98370, 32756, 42] avg=65536.0000 max=163920 max_rank=2 min=42 min_rank=7 imbalance_ratio=2.501221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[98196, 98103, 163920, 32762, 139, 98370, 32756, 42] avg=65536.0000 max=163920 max_rank=2 min=42 min_rank=7 imbalance_ratio=2.501221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[98196, 98103, 163920, 32762, 139, 98370, 32756, 42] avg=65536.0000 max=163920 max_rank=2 min=42 min_rank=7 imbalance_ratio=2.501221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65579, 66192, 65551, 66192, 66192, 64275, 66192, 64115] avg=65536.0000 max=66192 max_rank=1 min=64115 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32149 max_load_reduction_pct=0.326913 moved_to_replicas=191176 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[32769, 65533, 70939, 65546, 98286, 70149, 32813, 88253] avg=65536.0000 max=98286 max_rank=4 min=32769 min_rank=0 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[32769, 65533, 70939, 65546, 98286, 70149, 32813, 88253] avg=65536.0000 max=98286 max_rank=4 min=32769 min_rank=0 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] avg=65536.0000 max=66192 max_rank=0 min=64805 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97728 max_load_reduction_pct=0.596193 moved_to_replicas=387741 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] avg=65536.0000 max=66192 max_rank=0 min=64805 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97728 max_load_reduction_pct=0.596193 moved_to_replicas=387741 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[32769, 65533, 70939, 65546, 98286, 70149, 32813, 88253] avg=65536.0000 max=98286 max_rank=4 min=32769 min_rank=0 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] avg=65536.0000 max=66192 max_rank=0 min=64805 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97728 max_load_reduction_pct=0.596193 moved_to_replicas=387741 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] avg=65536.0000 max=66192 max_rank=0 min=64805 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97728 max_load_reduction_pct=0.596193 moved_to_replicas=387741 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] avg=65536.0000 max=66193 max_rank=7 min=63579 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32093 max_load_reduction_pct=0.326527 moved_to_replicas=125713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] avg=65536.0000 max=66193 max_rank=7 min=63579 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32093 max_load_reduction_pct=0.326527 moved_to_replicas=125713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[98196, 98103, 163920, 32762, 139, 98370, 32756, 42] avg=65536.0000 max=163920 max_rank=2 min=42 min_rank=7 imbalance_ratio=2.501221 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] avg=65536.0000 max=66193 max_rank=7 min=63579 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32093 max_load_reduction_pct=0.326527 moved_to_replicas=125713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[32769, 65533, 70939, 65546, 98286, 70149, 32813, 88253] avg=65536.0000 max=98286 max_rank=4 min=32769 min_rank=0 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[32769, 65533, 70939, 65546, 98286, 70149, 32813, 88253] avg=65536.0000 max=98286 max_rank=4 min=32769 min_rank=0 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[32769, 65533, 70939, 65546, 98286, 70149, 32813, 88253] avg=65536.0000 max=98286 max_rank=4 min=32769 min_rank=0 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[32769, 65533, 70939, 65546, 98286, 70149, 32813, 88253] avg=65536.0000 max=98286 max_rank=4 min=32769 min_rank=0 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131041, 32810, 35, 98249, 32678, 33626, 163055] avg=65536.0000 max=163055 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.488022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] avg=65536.0000 max=66193 max_rank=7 min=63579 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32093 max_load_reduction_pct=0.326527 moved_to_replicas=125713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131041, 32810, 35, 98249, 32678, 33626, 163055] avg=65536.0000 max=163055 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.488022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64940, 64808, 66192, 64967, 64805] avg=65536.0000 max=66192 max_rank=0 min=64805 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97728 max_load_reduction_pct=0.596193 moved_to_replicas=387741 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] avg=65536.0000 max=66193 max_rank=7 min=63579 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32093 max_load_reduction_pct=0.326527 moved_to_replicas=125713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131041, 32810, 35, 98249, 32678, 33626, 163055] avg=65536.0000 max=163055 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.488022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] avg=65536.0000 max=66193 max_rank=7 min=63579 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32093 max_load_reduction_pct=0.326527 moved_to_replicas=125713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] avg=65536.0000 max=66193 max_rank=7 min=63579 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32093 max_load_reduction_pct=0.326527 moved_to_replicas=125713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] avg=65536.0000 max=66192 max_rank=1 min=64142 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96863 max_load_reduction_pct=0.594051 moved_to_replicas=387805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] avg=65536.0000 max=66192 max_rank=1 min=64142 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96863 max_load_reduction_pct=0.594051 moved_to_replicas=387805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] avg=65536.0000 max=66192 max_rank=1 min=64142 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96863 max_load_reduction_pct=0.594051 moved_to_replicas=387805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131041, 32810, 35, 98249, 32678, 33626, 163055] avg=65536.0000 max=163055 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.488022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131041, 32810, 35, 98249, 32678, 33626, 163055] avg=65536.0000 max=163055 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.488022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[32769, 65533, 70939, 65546, 98286, 70149, 32813, 88253] avg=65536.0000 max=98286 max_rank=4 min=32769 min_rank=0 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131041, 32810, 35, 98249, 32678, 33626, 163055] avg=65536.0000 max=163055 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.488022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131041, 32810, 35, 98249, 32678, 33626, 163055] avg=65536.0000 max=163055 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.488022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[98249, 32813, 130965, 65554, 62, 32817, 65838, 97990] avg=65536.0000 max=130965 max_rank=2 min=62 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[98249, 32813, 130965, 65554, 62, 32817, 65838, 97990] avg=65536.0000 max=130965 max_rank=2 min=62 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] avg=65536.0000 max=66192 max_rank=1 min=64142 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96863 max_load_reduction_pct=0.594051 moved_to_replicas=387805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] avg=65536.0000 max=66192 max_rank=1 min=64142 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96863 max_load_reduction_pct=0.594051 moved_to_replicas=387805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[98249, 32813, 130965, 65554, 62, 32817, 65838, 97990] avg=65536.0000 max=130965 max_rank=2 min=62 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[64863, 65533, 66190, 65546, 66192, 66192, 63579, 66193] avg=65536.0000 max=66193 max_rank=7 min=63579 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32093 max_load_reduction_pct=0.326527 moved_to_replicas=125713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] avg=65536.0000 max=66192 max_rank=1 min=64142 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96863 max_load_reduction_pct=0.594051 moved_to_replicas=387805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] avg=65536.0000 max=66192 max_rank=1 min=64142 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96863 max_load_reduction_pct=0.594051 moved_to_replicas=387805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] avg=65536.0000 max=66192 max_rank=0 min=64616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257337 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] avg=65536.0000 max=66192 max_rank=0 min=64616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257337 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] avg=65536.0000 max=66192 max_rank=0 min=64616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257337 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[98249, 32813, 130965, 65554, 62, 32817, 65838, 97990] avg=65536.0000 max=130965 max_rank=2 min=62 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[98249, 32813, 130965, 65554, 62, 32817, 65838, 97990] avg=65536.0000 max=130965 max_rank=2 min=62 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131041, 32810, 35, 98249, 32678, 33626, 163055] avg=65536.0000 max=163055 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.488022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[98249, 32813, 130965, 65554, 62, 32817, 65838, 97990] avg=65536.0000 max=130965 max_rank=2 min=62 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[98249, 32813, 130965, 65554, 62, 32817, 65838, 97990] avg=65536.0000 max=130965 max_rank=2 min=62 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[65531, 10721, 130601, 65535, 32805, 88042, 98282, 32771] avg=65536.0000 max=130601 max_rank=2 min=10721 min_rank=1 imbalance_ratio=1.992813 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[65531, 10721, 130601, 65535, 32805, 88042, 98282, 32771] avg=65536.0000 max=130601 max_rank=2 min=10721 min_rank=1 imbalance_ratio=1.992813 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] avg=65536.0000 max=66192 max_rank=0 min=64616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257337 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] avg=65536.0000 max=66192 max_rank=0 min=64616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257337 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[65531, 10721, 130601, 65535, 32805, 88042, 98282, 32771] avg=65536.0000 max=130601 max_rank=2 min=10721 min_rank=1 imbalance_ratio=1.992813 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64904, 64142, 66192, 65434, 65683, 66192] avg=65536.0000 max=66192 max_rank=1 min=64142 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96863 max_load_reduction_pct=0.594051 moved_to_replicas=387805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] avg=65536.0000 max=66192 max_rank=0 min=64616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257337 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] avg=65536.0000 max=66192 max_rank=0 min=64616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257337 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64409 max_load_reduction_pct=0.493174 moved_to_replicas=238459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64409 max_load_reduction_pct=0.493174 moved_to_replicas=238459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64409 max_load_reduction_pct=0.493174 moved_to_replicas=238459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[65531, 10721, 130601, 65535, 32805, 88042, 98282, 32771] avg=65536.0000 max=130601 max_rank=2 min=10721 min_rank=1 imbalance_ratio=1.992813 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[65531, 10721, 130601, 65535, 32805, 88042, 98282, 32771] avg=65536.0000 max=130601 max_rank=2 min=10721 min_rank=1 imbalance_ratio=1.992813 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[98249, 32813, 130965, 65554, 62, 32817, 65838, 97990] avg=65536.0000 max=130965 max_rank=2 min=62 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[65531, 10721, 130601, 65535, 32805, 88042, 98282, 32771] avg=65536.0000 max=130601 max_rank=2 min=10721 min_rank=1 imbalance_ratio=1.992813 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[65531, 10721, 130601, 65535, 32805, 88042, 98282, 32771] avg=65536.0000 max=130601 max_rank=2 min=10721 min_rank=1 imbalance_ratio=1.992813 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32560, 32790, 131045, 137816, 65440, 65545, 32792, 26300] avg=65536.0000 max=137816 max_rank=3 min=26300 min_rank=7 imbalance_ratio=2.102905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32560, 32790, 131045, 137816, 65440, 65545, 32792, 26300] avg=65536.0000 max=137816 max_rank=3 min=26300 min_rank=7 imbalance_ratio=2.102905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64409 max_load_reduction_pct=0.493174 moved_to_replicas=238459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64830, 66192, 65554, 64616, 64874, 65838, 66192] avg=65536.0000 max=66192 max_rank=0 min=64616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257337 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64409 max_load_reduction_pct=0.493174 moved_to_replicas=238459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32560, 32790, 131045, 137816, 65440, 65545, 32792, 26300] avg=65536.0000 max=137816 max_rank=3 min=26300 min_rank=7 imbalance_ratio=2.102905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64409 max_load_reduction_pct=0.493174 moved_to_replicas=238459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64409 max_load_reduction_pct=0.493174 moved_to_replicas=238459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] avg=65536.0000 max=66195 max_rank=1 min=63865 min_rank=7 imbalance_ratio=1.010056 max_load_reduction=71621 max_load_reduction_pct=0.519686 moved_to_replicas=272791 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] avg=65536.0000 max=66195 max_rank=1 min=63865 min_rank=7 imbalance_ratio=1.010056 max_load_reduction=71621 max_load_reduction_pct=0.519686 moved_to_replicas=272791 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] avg=65536.0000 max=66195 max_rank=1 min=63865 min_rank=7 imbalance_ratio=1.010056 max_load_reduction=71621 max_load_reduction_pct=0.519686 moved_to_replicas=272791 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32560, 32790, 131045, 137816, 65440, 65545, 32792, 26300] avg=65536.0000 max=137816 max_rank=3 min=26300 min_rank=7 imbalance_ratio=2.102905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[65531, 10721, 130601, 65535, 32805, 88042, 98282, 32771] avg=65536.0000 max=130601 max_rank=2 min=10721 min_rank=1 imbalance_ratio=1.992813 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32560, 32790, 131045, 137816, 65440, 65545, 32792, 26300] avg=65536.0000 max=137816 max_rank=3 min=26300 min_rank=7 imbalance_ratio=2.102905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32560, 32790, 131045, 137816, 65440, 65545, 32792, 26300] avg=65536.0000 max=137816 max_rank=3 min=26300 min_rank=7 imbalance_ratio=2.102905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32560, 32790, 131045, 137816, 65440, 65545, 32792, 26300] avg=65536.0000 max=137816 max_rank=3 min=26300 min_rank=7 imbalance_ratio=2.102905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] avg=65536.0000 max=66195 max_rank=1 min=63865 min_rank=7 imbalance_ratio=1.010056 max_load_reduction=71621 max_load_reduction_pct=0.519686 moved_to_replicas=272791 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65328, 66192, 65535, 64895, 66190, 66192, 64425] avg=65536.0000 max=66192 max_rank=2 min=64425 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64409 max_load_reduction_pct=0.493174 moved_to_replicas=238459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] avg=65536.0000 max=66195 max_rank=1 min=63865 min_rank=7 imbalance_ratio=1.010056 max_load_reduction=71621 max_load_reduction_pct=0.519686 moved_to_replicas=272791 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] avg=65536.0000 max=66195 max_rank=1 min=63865 min_rank=7 imbalance_ratio=1.010056 max_load_reduction=71621 max_load_reduction_pct=0.519686 moved_to_replicas=272791 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] avg=65536.0000 max=66195 max_rank=1 min=63865 min_rank=7 imbalance_ratio=1.010056 max_load_reduction=71621 max_load_reduction_pct=0.519686 moved_to_replicas=272791 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32560, 32790, 131045, 137816, 65440, 65545, 32792, 26300] avg=65536.0000 max=137816 max_rank=3 min=26300 min_rank=7 imbalance_ratio=2.102905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[65314, 66195, 66192, 66191, 65440, 65545, 65546, 63865] avg=65536.0000 max=66195 max_rank=1 min=63865 min_rank=7 imbalance_ratio=1.010056 max_load_reduction=71621 max_load_reduction_pct=0.519686 moved_to_replicas=272791 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=2105242 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=103552 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=2087491 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=119680 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=2104422 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=105088 peak_demand=66195 required_capacity=1.010056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=2100821 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=109696 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=2095147 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=112512 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=2097163 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=111616 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=2090842 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=116224 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=2096088 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=112256 peak_demand=66193 required_capacity=1.010025 + [2026-08-30 05:15:25] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 533.9 | throughput per GPU (TFLOP/s/GPU): 624.4 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.038030E+01 | loss scale: 1.0 | grad norm: 32.687 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=69632 expected_recv=125445 expected_overflow=0 exact_recv_per_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=69632 expected_recv=128828 expected_overflow=0 exact_recv_per_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=69632 expected_recv=125445 actual_valid_recv=127104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=69632 expected_recv=128828 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=127104 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=130048 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=69632 expected_recv=131155 expected_overflow=0 exact_recv_per_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=69632 expected_recv=128449 expected_overflow=0 exact_recv_per_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=69632 expected_recv=129309 expected_overflow=0 exact_recv_per_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=69632 expected_recv=131155 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=69632 expected_recv=130127 expected_overflow=0 exact_recv_per_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=69632 expected_recv=128449 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=69632 expected_recv=129309 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=69632 expected_recv=130127 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=69632 expected_recv=131083 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=69632 expected_recv=131076 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=69632 expected_recv=127219 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=134272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=69632 expected_recv=129661 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=69632 expected_recv=131083 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=69632 expected_recv=131076 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=134272 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=69632 expected_recv=127219 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=69632 expected_recv=129661 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=128768 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=69632 expected_recv=130384 expected_overflow=0 exact_recv_per_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=69632 expected_recv=130143 expected_overflow=0 exact_recv_per_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=69632 expected_recv=130020 expected_overflow=0 exact_recv_per_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=69632 expected_recv=130384 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=69632 expected_recv=130143 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=69632 expected_recv=130020 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=69632 expected_recv=130918 expected_overflow=0 exact_recv_per_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=69632 expected_recv=129959 expected_overflow=0 exact_recv_per_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=69632 expected_recv=130918 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=69632 expected_recv=129959 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=69632 expected_recv=131079 expected_overflow=0 exact_recv_per_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=69632 expected_recv=129802 expected_overflow=0 exact_recv_per_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=69632 expected_recv=131079 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=69632 expected_recv=129802 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=69632 expected_recv=129689 expected_overflow=0 exact_recv_per_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=69632 expected_recv=130410 expected_overflow=0 exact_recv_per_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=69632 expected_recv=130445 expected_overflow=0 exact_recv_per_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=69632 expected_recv=129689 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=69632 expected_recv=130410 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=69632 expected_recv=130445 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=69632 expected_recv=131213 expected_overflow=0 exact_recv_per_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=69632 expected_recv=126700 expected_overflow=0 exact_recv_per_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=69632 expected_recv=131213 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=69632 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=69632 expected_recv=126700 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=69632 expected_recv=130012 expected_overflow=0 exact_recv_per_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=128384 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=69632 expected_recv=132387 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=69632 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=69632 expected_recv=131114 expected_overflow=0 exact_recv_per_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=69632 expected_recv=130012 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=69632 expected_recv=132381 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=69632 expected_recv=131114 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=69632 expected_recv=129324 expected_overflow=0 exact_recv_per_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=69632 expected_recv=129324 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=69632 expected_recv=131037 expected_overflow=0 exact_recv_per_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=69632 expected_recv=131251 expected_overflow=0 exact_recv_per_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=69632 expected_recv=131037 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=69632 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=69632 expected_recv=131251 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=69632 expected_recv=130526 expected_overflow=0 exact_recv_per_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=69632 expected_recv=129282 expected_overflow=0 exact_recv_per_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=69632 expected_recv=132387 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=69632 expected_recv=130526 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=69632 expected_recv=129282 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=69632 expected_recv=131406 expected_overflow=0 exact_recv_per_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=69632 expected_recv=131406 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=69632 expected_recv=125448 expected_overflow=0 exact_recv_per_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=69632 expected_recv=125448 actual_valid_recv=126720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=69632 expected_recv=131118 expected_overflow=0 exact_recv_per_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=126720 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=69632 expected_recv=131068 expected_overflow=0 exact_recv_per_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=69632 expected_recv=131118 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=69632 expected_recv=130916 expected_overflow=0 exact_recv_per_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=69632 expected_recv=131068 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=69632 expected_recv=130916 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=69632 expected_recv=124670 expected_overflow=0 exact_recv_per_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=69632 expected_recv=124670 actual_valid_recv=125824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=125824 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=69632 expected_recv=131070 expected_overflow=0 exact_recv_per_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=69632 expected_recv=131070 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=69632 expected_recv=131035 expected_overflow=0 exact_recv_per_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=69632 expected_recv=129833 expected_overflow=0 exact_recv_per_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=69632 expected_recv=131035 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=69632 expected_recv=129833 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=69632 expected_recv=130881 expected_overflow=0 exact_recv_per_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=69632 expected_recv=130005 expected_overflow=0 exact_recv_per_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=69632 expected_recv=131056 expected_overflow=0 exact_recv_per_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=69632 expected_recv=130881 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=69632 expected_recv=130005 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=69632 expected_recv=131056 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=69632 expected_recv=130998 expected_overflow=0 exact_recv_per_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=69632 expected_recv=131077 expected_overflow=0 exact_recv_per_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=69632 expected_recv=130998 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=69632 expected_recv=129700 expected_overflow=0 exact_recv_per_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=69632 expected_recv=131077 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=69632 expected_recv=129700 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=69632 expected_recv=131079 expected_overflow=0 exact_recv_per_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=69632 expected_recv=131079 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=69632 expected_recv=129769 expected_overflow=0 exact_recv_per_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=69632 expected_recv=131164 expected_overflow=0 exact_recv_per_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=69632 expected_recv=131019 expected_overflow=0 exact_recv_per_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=69632 expected_recv=129769 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=69632 expected_recv=131164 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=69632 expected_recv=131019 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=69632 expected_recv=129876 expected_overflow=0 exact_recv_per_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=69632 expected_recv=131130 expected_overflow=0 exact_recv_per_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=69632 expected_recv=129876 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=69632 expected_recv=131113 expected_overflow=0 exact_recv_per_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=69632 expected_recv=131130 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=69632 expected_recv=131113 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=69632 expected_recv=129767 expected_overflow=0 exact_recv_per_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=69632 expected_recv=129767 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=69632 expected_recv=129538 expected_overflow=0 exact_recv_per_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=69632 expected_recv=129538 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=69632 expected_recv=129628 expected_overflow=0 exact_recv_per_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=69632 expected_recv=129628 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=69632 expected_recv=131087 expected_overflow=0 exact_recv_per_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=69632 expected_recv=131087 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=69632 expected_recv=129841 expected_overflow=0 exact_recv_per_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=69632 expected_recv=131163 expected_overflow=0 exact_recv_per_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=69632 expected_recv=129841 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=69632 expected_recv=131163 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=69632 expected_recv=125648 expected_overflow=0 exact_recv_per_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=69632 expected_recv=130985 expected_overflow=0 exact_recv_per_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=69632 expected_recv=131104 expected_overflow=0 exact_recv_per_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=69632 expected_recv=125648 actual_valid_recv=126720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=69632 expected_recv=130985 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=69632 expected_recv=131104 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=126720 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=134144 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=69632 expected_recv=131063 expected_overflow=0 exact_recv_per_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=69632 expected_recv=131156 expected_overflow=0 exact_recv_per_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=69632 expected_recv=131063 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=69632 expected_recv=131156 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=69632 expected_recv=128371 expected_overflow=0 exact_recv_per_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=69632 expected_recv=129692 expected_overflow=0 exact_recv_per_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=69632 expected_recv=131174 expected_overflow=0 exact_recv_per_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=69632 expected_recv=128371 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=69632 expected_recv=129692 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=69632 expected_recv=131174 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=69632 expected_recv=131133 expected_overflow=0 exact_recv_per_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=69632 expected_recv=131133 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=69632 expected_recv=131177 expected_overflow=0 exact_recv_per_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=69632 expected_recv=131081 expected_overflow=0 exact_recv_per_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=69632 expected_recv=128272 expected_overflow=0 exact_recv_per_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=69632 expected_recv=129847 expected_overflow=0 exact_recv_per_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=69632 expected_recv=131177 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=69632 expected_recv=131081 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=69632 expected_recv=129861 expected_overflow=0 exact_recv_per_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=69632 expected_recv=128272 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=69632 expected_recv=129847 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=69632 expected_recv=129861 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=129536 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[17411, 89189, 87281, 44809, 78571, 66428, 76943, 63656] avg=65536.0000 max=89189 max_rank=1 min=17411 min_rank=0 imbalance_ratio=1.360916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[17411, 89189, 87281, 44809, 78571, 66428, 76943, 63656] avg=65536.0000 max=89189 max_rank=1 min=17411 min_rank=0 imbalance_ratio=1.360916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[17411, 89189, 87281, 44809, 78571, 66428, 76943, 63656] avg=65536.0000 max=89189 max_rank=1 min=17411 min_rank=0 imbalance_ratio=1.360916 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=69632 expected_recv=131033 expected_overflow=0 exact_recv_per_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[17411, 89189, 87281, 44809, 78571, 66428, 76943, 63656] avg=65536.0000 max=89189 max_rank=1 min=17411 min_rank=0 imbalance_ratio=1.360916 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=69632 expected_recv=131033 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] avg=65536.0000 max=66192 max_rank=1 min=63480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22997 max_load_reduction_pct=0.257846 moved_to_replicas=133826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] avg=65536.0000 max=66192 max_rank=1 min=63480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22997 max_load_reduction_pct=0.257846 moved_to_replicas=133826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[17411, 89189, 87281, 44809, 78571, 66428, 76943, 63656] avg=65536.0000 max=89189 max_rank=1 min=17411 min_rank=0 imbalance_ratio=1.360916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[17411, 89189, 87281, 44809, 78571, 66428, 76943, 63656] avg=65536.0000 max=89189 max_rank=1 min=17411 min_rank=0 imbalance_ratio=1.360916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] avg=65536.0000 max=66192 max_rank=1 min=63480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22997 max_load_reduction_pct=0.257846 moved_to_replicas=133826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] avg=65536.0000 max=66192 max_rank=1 min=63480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22997 max_load_reduction_pct=0.257846 moved_to_replicas=133826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] avg=65536.0000 max=66192 max_rank=1 min=63480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22997 max_load_reduction_pct=0.257846 moved_to_replicas=133826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] avg=65536.0000 max=66192 max_rank=1 min=63480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22997 max_load_reduction_pct=0.257846 moved_to_replicas=133826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[99403, 59672, 64916, 68614, 99276, 64019, 35609, 32779] avg=65536.0000 max=99403 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.516769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[99403, 59672, 64916, 68614, 99276, 64019, 35609, 32779] avg=65536.0000 max=99403 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.516769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[99403, 59672, 64916, 68614, 99276, 64019, 35609, 32779] avg=65536.0000 max=99403 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.516769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[17411, 89189, 87281, 44809, 78571, 66428, 76943, 63656] avg=65536.0000 max=89189 max_rank=1 min=17411 min_rank=0 imbalance_ratio=1.360916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[17411, 89189, 87281, 44809, 78571, 66428, 76943, 63656] avg=65536.0000 max=89189 max_rank=1 min=17411 min_rank=0 imbalance_ratio=1.360916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[99403, 59672, 64916, 68614, 99276, 64019, 35609, 32779] avg=65536.0000 max=99403 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.516769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] avg=65536.0000 max=66192 max_rank=0 min=64019 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33211 max_load_reduction_pct=0.334105 moved_to_replicas=138732 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] avg=65536.0000 max=66192 max_rank=0 min=64019 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33211 max_load_reduction_pct=0.334105 moved_to_replicas=138732 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[99403, 59672, 64916, 68614, 99276, 64019, 35609, 32779] avg=65536.0000 max=99403 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.516769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[99403, 59672, 64916, 68614, 99276, 64019, 35609, 32779] avg=65536.0000 max=99403 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.516769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] avg=65536.0000 max=66192 max_rank=0 min=64019 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33211 max_load_reduction_pct=0.334105 moved_to_replicas=138732 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] avg=65536.0000 max=66192 max_rank=1 min=63480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22997 max_load_reduction_pct=0.257846 moved_to_replicas=133826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] avg=65536.0000 max=66192 max_rank=0 min=64019 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33211 max_load_reduction_pct=0.334105 moved_to_replicas=138732 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[63480, 66192, 66192, 66192, 66192, 66192, 66192, 63656] avg=65536.0000 max=66192 max_rank=1 min=63480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=22997 max_load_reduction_pct=0.257846 moved_to_replicas=133826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] avg=65536.0000 max=66192 max_rank=0 min=64019 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33211 max_load_reduction_pct=0.334105 moved_to_replicas=138732 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] avg=65536.0000 max=66192 max_rank=0 min=64019 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33211 max_load_reduction_pct=0.334105 moved_to_replicas=138732 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65534, 16400, 54295, 32999, 98293, 93095, 65542, 98130] avg=65536.0000 max=98293 max_rank=4 min=16400 min_rank=1 imbalance_ratio=1.499832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65534, 16400, 54295, 32999, 98293, 93095, 65542, 98130] avg=65536.0000 max=98293 max_rank=4 min=16400 min_rank=1 imbalance_ratio=1.499832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65534, 16400, 54295, 32999, 98293, 93095, 65542, 98130] avg=65536.0000 max=98293 max_rank=4 min=16400 min_rank=1 imbalance_ratio=1.499832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[99403, 59672, 64916, 68614, 99276, 64019, 35609, 32779] avg=65536.0000 max=99403 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.516769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65534, 16400, 54295, 32999, 98293, 93095, 65542, 98130] avg=65536.0000 max=98293 max_rank=4 min=16400 min_rank=1 imbalance_ratio=1.499832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[99403, 59672, 64916, 68614, 99276, 64019, 35609, 32779] avg=65536.0000 max=99403 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.516769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] avg=65536.0000 max=66193 max_rank=5 min=63506 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32100 max_load_reduction_pct=0.326575 moved_to_replicas=182776 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] avg=65536.0000 max=66193 max_rank=5 min=63506 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32100 max_load_reduction_pct=0.326575 moved_to_replicas=182776 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] avg=65536.0000 max=66193 max_rank=5 min=63506 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32100 max_load_reduction_pct=0.326575 moved_to_replicas=182776 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65534, 16400, 54295, 32999, 98293, 93095, 65542, 98130] avg=65536.0000 max=98293 max_rank=4 min=16400 min_rank=1 imbalance_ratio=1.499832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65534, 16400, 54295, 32999, 98293, 93095, 65542, 98130] avg=65536.0000 max=98293 max_rank=4 min=16400 min_rank=1 imbalance_ratio=1.499832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] avg=65536.0000 max=66192 max_rank=0 min=64019 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33211 max_load_reduction_pct=0.334105 moved_to_replicas=138732 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] avg=65536.0000 max=66193 max_rank=5 min=63506 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32100 max_load_reduction_pct=0.326575 moved_to_replicas=182776 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65041, 64916, 66192, 66192, 64019, 66192, 65544] avg=65536.0000 max=66192 max_rank=0 min=64019 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33211 max_load_reduction_pct=0.334105 moved_to_replicas=138732 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] avg=65536.0000 max=66193 max_rank=5 min=63506 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32100 max_load_reduction_pct=0.326575 moved_to_replicas=182776 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] avg=65536.0000 max=66193 max_rank=5 min=63506 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32100 max_load_reduction_pct=0.326575 moved_to_replicas=182776 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65498, 32862, 98239, 98539, 163800, 65298] avg=65536.0000 max=163800 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65498, 32862, 98239, 98539, 163800, 65298] avg=65536.0000 max=163800 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65498, 32862, 98239, 98539, 163800, 65298] avg=65536.0000 max=163800 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65534, 16400, 54295, 32999, 98293, 93095, 65542, 98130] avg=65536.0000 max=98293 max_rank=4 min=16400 min_rank=1 imbalance_ratio=1.499832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65534, 16400, 54295, 32999, 98293, 93095, 65542, 98130] avg=65536.0000 max=98293 max_rank=4 min=16400 min_rank=1 imbalance_ratio=1.499832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65498, 32862, 98239, 98539, 163800, 65298] avg=65536.0000 max=163800 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] avg=65536.0000 max=66192 max_rank=4 min=64864 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97608 max_load_reduction_pct=0.595897 moved_to_replicas=324212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] avg=65536.0000 max=66192 max_rank=4 min=64864 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97608 max_load_reduction_pct=0.595897 moved_to_replicas=324212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] avg=65536.0000 max=66192 max_rank=4 min=64864 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97608 max_load_reduction_pct=0.595897 moved_to_replicas=324212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65498, 32862, 98239, 98539, 163800, 65298] avg=65536.0000 max=163800 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65498, 32862, 98239, 98539, 163800, 65298] avg=65536.0000 max=163800 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] avg=65536.0000 max=66193 max_rank=5 min=63506 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32100 max_load_reduction_pct=0.326575 moved_to_replicas=182776 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66192, 63506, 64937, 66192, 66193, 65542, 66192] avg=65536.0000 max=66193 max_rank=5 min=63506 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32100 max_load_reduction_pct=0.326575 moved_to_replicas=182776 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] avg=65536.0000 max=66192 max_rank=4 min=64864 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97608 max_load_reduction_pct=0.595897 moved_to_replicas=324212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] avg=65536.0000 max=66192 max_rank=4 min=64864 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97608 max_load_reduction_pct=0.595897 moved_to_replicas=324212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] avg=65536.0000 max=66192 max_rank=4 min=64864 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97608 max_load_reduction_pct=0.595897 moved_to_replicas=324212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[94118, 65540, 51848, 98278, 98307, 50637, 32787, 32773] avg=65536.0000 max=98307 max_rank=4 min=32773 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[94118, 65540, 51848, 98278, 98307, 50637, 32787, 32773] avg=65536.0000 max=98307 max_rank=4 min=32773 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[94118, 65540, 51848, 98278, 98307, 50637, 32787, 32773] avg=65536.0000 max=98307 max_rank=4 min=32773 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65498, 32862, 98239, 98539, 163800, 65298] avg=65536.0000 max=163800 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=185755 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65498, 32862, 98239, 98539, 163800, 65298] avg=65536.0000 max=163800 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=185755 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=185755 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[94118, 65540, 51848, 98278, 98307, 50637, 32787, 32773] avg=65536.0000 max=98307 max_rank=4 min=32773 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[94118, 65540, 51848, 98278, 98307, 50637, 32787, 32773] avg=65536.0000 max=98307 max_rank=4 min=32773 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[94118, 65540, 51848, 98278, 98307, 50637, 32787, 32773] avg=65536.0000 max=98307 max_rank=4 min=32773 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] avg=65536.0000 max=66192 max_rank=4 min=64864 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97608 max_load_reduction_pct=0.595897 moved_to_replicas=324212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65143, 64864, 65498, 64909, 66192, 66192, 66192, 65298] avg=65536.0000 max=66192 max_rank=4 min=64864 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97608 max_load_reduction_pct=0.595897 moved_to_replicas=324212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=185755 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=185755 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=185755 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[54247, 65594, 93674, 81126, 65545, 98418, 32792, 32892] avg=65536.0000 max=98418 max_rank=5 min=32792 min_rank=6 imbalance_ratio=1.501740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[54247, 65594, 93674, 81126, 65545, 98418, 32792, 32892] avg=65536.0000 max=98418 max_rank=5 min=32792 min_rank=6 imbalance_ratio=1.501740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[54247, 65594, 93674, 81126, 65545, 98418, 32792, 32892] avg=65536.0000 max=98418 max_rank=5 min=32792 min_rank=6 imbalance_ratio=1.501740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[94118, 65540, 51848, 98278, 98307, 50637, 32787, 32773] avg=65536.0000 max=98307 max_rank=4 min=32773 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] avg=65536.0000 max=66194 max_rank=0 min=63362 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.327420 moved_to_replicas=149857 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[94118, 65540, 51848, 98278, 98307, 50637, 32787, 32773] avg=65536.0000 max=98307 max_rank=4 min=32773 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] avg=65536.0000 max=66194 max_rank=0 min=63362 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.327420 moved_to_replicas=149857 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] avg=65536.0000 max=66194 max_rank=0 min=63362 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.327420 moved_to_replicas=149857 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[54247, 65594, 93674, 81126, 65545, 98418, 32792, 32892] avg=65536.0000 max=98418 max_rank=5 min=32792 min_rank=6 imbalance_ratio=1.501740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[54247, 65594, 93674, 81126, 65545, 98418, 32792, 32892] avg=65536.0000 max=98418 max_rank=5 min=32792 min_rank=6 imbalance_ratio=1.501740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=185755 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[54247, 65594, 93674, 81126, 65545, 98418, 32792, 32892] avg=65536.0000 max=98418 max_rank=5 min=32792 min_rank=6 imbalance_ratio=1.501740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[66191, 65540, 64218, 66192, 66192, 66194, 64873, 64888] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=185755 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] avg=65536.0000 max=66194 max_rank=0 min=63362 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.327420 moved_to_replicas=149857 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] avg=65536.0000 max=66194 max_rank=0 min=63362 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.327420 moved_to_replicas=149857 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] avg=65536.0000 max=66194 max_rank=0 min=63362 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.327420 moved_to_replicas=149857 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32783, 65326, 9045, 32783, 147677, 65577, 105563] avg=65536.0000 max=147677 max_rank=5 min=9045 min_rank=3 imbalance_ratio=2.253372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32783, 65326, 9045, 32783, 147677, 65577, 105563] avg=65536.0000 max=147677 max_rank=5 min=9045 min_rank=3 imbalance_ratio=2.253372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32783, 65326, 9045, 32783, 147677, 65577, 105563] avg=65536.0000 max=147677 max_rank=5 min=9045 min_rank=3 imbalance_ratio=2.253372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[54247, 65594, 93674, 81126, 65545, 98418, 32792, 32892] avg=65536.0000 max=98418 max_rank=5 min=32792 min_rank=6 imbalance_ratio=1.501740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] avg=65536.0000 max=66193 max_rank=3 min=63732 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=81484 max_load_reduction_pct=0.551772 moved_to_replicas=238435 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] avg=65536.0000 max=66193 max_rank=3 min=63732 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=81484 max_load_reduction_pct=0.551772 moved_to_replicas=238435 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] avg=65536.0000 max=66193 max_rank=3 min=63732 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=81484 max_load_reduction_pct=0.551772 moved_to_replicas=238435 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32783, 65326, 9045, 32783, 147677, 65577, 105563] avg=65536.0000 max=147677 max_rank=5 min=9045 min_rank=3 imbalance_ratio=2.253372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[54247, 65594, 93674, 81126, 65545, 98418, 32792, 32892] avg=65536.0000 max=98418 max_rank=5 min=32792 min_rank=6 imbalance_ratio=1.501740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32783, 65326, 9045, 32783, 147677, 65577, 105563] avg=65536.0000 max=147677 max_rank=5 min=9045 min_rank=3 imbalance_ratio=2.253372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] avg=65536.0000 max=66194 max_rank=0 min=63362 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.327420 moved_to_replicas=149857 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32783, 65326, 9045, 32783, 147677, 65577, 105563] avg=65536.0000 max=147677 max_rank=5 min=9045 min_rank=3 imbalance_ratio=2.253372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] avg=65536.0000 max=66193 max_rank=3 min=63732 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=81484 max_load_reduction_pct=0.551772 moved_to_replicas=238435 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[66194, 65594, 66193, 66190, 65545, 66192, 65018, 63362] avg=65536.0000 max=66194 max_rank=0 min=63362 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32224 max_load_reduction_pct=0.327420 moved_to_replicas=149857 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] avg=65536.0000 max=66193 max_rank=3 min=63732 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=81484 max_load_reduction_pct=0.551772 moved_to_replicas=238435 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[61114, 32912, 143863, 65560, 56992, 15, 32777, 131055] avg=65536.0000 max=143863 max_rank=2 min=15 min_rank=5 imbalance_ratio=2.195175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[61114, 32912, 143863, 65560, 56992, 15, 32777, 131055] avg=65536.0000 max=143863 max_rank=2 min=15 min_rank=5 imbalance_ratio=2.195175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] avg=65536.0000 max=66193 max_rank=3 min=63732 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=81484 max_load_reduction_pct=0.551772 moved_to_replicas=238435 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[61114, 32912, 143863, 65560, 56992, 15, 32777, 131055] avg=65536.0000 max=143863 max_rank=2 min=15 min_rank=5 imbalance_ratio=2.195175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32783, 65326, 9045, 32783, 147677, 65577, 105563] avg=65536.0000 max=147677 max_rank=5 min=9045 min_rank=3 imbalance_ratio=2.253372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62753 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77670 max_load_reduction_pct=0.539889 moved_to_replicas=285423 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62753 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77670 max_load_reduction_pct=0.539889 moved_to_replicas=285423 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[61114, 32912, 143863, 65560, 56992, 15, 32777, 131055] avg=65536.0000 max=143863 max_rank=2 min=15 min_rank=5 imbalance_ratio=2.195175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62753 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77670 max_load_reduction_pct=0.539889 moved_to_replicas=285423 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32783, 65326, 9045, 32783, 147677, 65577, 105563] avg=65536.0000 max=147677 max_rank=5 min=9045 min_rank=3 imbalance_ratio=2.253372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] avg=65536.0000 max=66193 max_rank=3 min=63732 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=81484 max_load_reduction_pct=0.551772 moved_to_replicas=238435 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[61114, 32912, 143863, 65560, 56992, 15, 32777, 131055] avg=65536.0000 max=143863 max_rank=2 min=15 min_rank=5 imbalance_ratio=2.195175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[61114, 32912, 143863, 65560, 56992, 15, 32777, 131055] avg=65536.0000 max=143863 max_rank=2 min=15 min_rank=5 imbalance_ratio=2.195175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62753 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77670 max_load_reduction_pct=0.539889 moved_to_replicas=285423 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65543, 65326, 66193, 63732, 66191, 65577, 66192] avg=65536.0000 max=66193 max_rank=3 min=63732 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=81484 max_load_reduction_pct=0.551772 moved_to_replicas=238435 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[58617, 65492, 23, 98321, 65535, 96910, 37420, 101970] avg=65536.0000 max=101970 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.555939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[58617, 65492, 23, 98321, 65535, 96910, 37420, 101970] avg=65536.0000 max=101970 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.555939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62753 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77670 max_load_reduction_pct=0.539889 moved_to_replicas=285423 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62753 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77670 max_load_reduction_pct=0.539889 moved_to_replicas=285423 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[58617, 65492, 23, 98321, 65535, 96910, 37420, 101970] avg=65536.0000 max=101970 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.555939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[61114, 32912, 143863, 65560, 56992, 15, 32777, 131055] avg=65536.0000 max=143863 max_rank=2 min=15 min_rank=5 imbalance_ratio=2.195175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62301 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=35778 max_load_reduction_pct=0.350868 moved_to_replicas=197423 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62301 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=35778 max_load_reduction_pct=0.350868 moved_to_replicas=197423 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[58617, 65492, 23, 98321, 65535, 96910, 37420, 101970] avg=65536.0000 max=101970 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.555939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[61114, 32912, 143863, 65560, 56992, 15, 32777, 131055] avg=65536.0000 max=143863 max_rank=2 min=15 min_rank=5 imbalance_ratio=2.195175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62301 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=35778 max_load_reduction_pct=0.350868 moved_to_replicas=197423 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[58617, 65492, 23, 98321, 65535, 96910, 37420, 101970] avg=65536.0000 max=101970 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.555939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62753 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77670 max_load_reduction_pct=0.539889 moved_to_replicas=285423 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[58617, 65492, 23, 98321, 65535, 96910, 37420, 101970] avg=65536.0000 max=101970 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.555939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62301 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=35778 max_load_reduction_pct=0.350868 moved_to_replicas=197423 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98373, 32791, 65447, 65538, 65532, 98297, 65518] avg=65536.0000 max=98373 max_rank=1 min=32791 min_rank=2 imbalance_ratio=1.501053 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[62753, 65673, 66192, 65560, 66193, 65534, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62753 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77670 max_load_reduction_pct=0.539889 moved_to_replicas=285423 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98373, 32791, 65447, 65538, 65532, 98297, 65518] avg=65536.0000 max=98373 max_rank=1 min=32791 min_rank=2 imbalance_ratio=1.501053 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62301 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=35778 max_load_reduction_pct=0.350868 moved_to_replicas=197423 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98373, 32791, 65447, 65538, 65532, 98297, 65518] avg=65536.0000 max=98373 max_rank=1 min=32791 min_rank=2 imbalance_ratio=1.501053 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62301 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=35778 max_load_reduction_pct=0.350868 moved_to_replicas=197423 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[58617, 65492, 23, 98321, 65535, 96910, 37420, 101970] avg=65536.0000 max=101970 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.555939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.327132 moved_to_replicas=128603 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98373, 32791, 65447, 65538, 65532, 98297, 65518] avg=65536.0000 max=98373 max_rank=1 min=32791 min_rank=2 imbalance_ratio=1.501053 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.327132 moved_to_replicas=128603 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.327132 moved_to_replicas=128603 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[58617, 65492, 23, 98321, 65535, 96910, 37420, 101970] avg=65536.0000 max=101970 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.555939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98373, 32791, 65447, 65538, 65532, 98297, 65518] avg=65536.0000 max=98373 max_rank=1 min=32791 min_rank=2 imbalance_ratio=1.501053 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62301 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=35778 max_load_reduction_pct=0.350868 moved_to_replicas=197423 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.327132 moved_to_replicas=128603 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98373, 32791, 65447, 65538, 65532, 98297, 65518] avg=65536.0000 max=98373 max_rank=1 min=32791 min_rank=2 imbalance_ratio=1.501053 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32776, 65577, 32883, 65518, 131042, 98152, 32781] avg=65536.0000 max=131042 max_rank=5 min=32776 min_rank=1 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32776, 65577, 32883, 65518, 131042, 98152, 32781] avg=65536.0000 max=131042 max_rank=5 min=32776 min_rank=1 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.327132 moved_to_replicas=128603 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[62301, 65492, 66192, 66192, 65535, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=62301 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=35778 max_load_reduction_pct=0.350868 moved_to_replicas=197423 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32776, 65577, 32883, 65518, 131042, 98152, 32781] avg=65536.0000 max=131042 max_rank=5 min=32776 min_rank=1 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.327132 moved_to_replicas=128603 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98373, 32791, 65447, 65538, 65532, 98297, 65518] avg=65536.0000 max=98373 max_rank=1 min=32791 min_rank=2 imbalance_ratio=1.501053 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] avg=65536.0000 max=66192 max_rank=5 min=64843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=193334 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32776, 65577, 32883, 65518, 131042, 98152, 32781] avg=65536.0000 max=131042 max_rank=5 min=32776 min_rank=1 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] avg=65536.0000 max=66192 max_rank=5 min=64843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=193334 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] avg=65536.0000 max=66192 max_rank=5 min=64843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=193334 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32776, 65577, 32883, 65518, 131042, 98152, 32781] avg=65536.0000 max=131042 max_rank=5 min=32776 min_rank=1 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98373, 32791, 65447, 65538, 65532, 98297, 65518] avg=65536.0000 max=98373 max_rank=1 min=32791 min_rank=2 imbalance_ratio=1.501053 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.327132 moved_to_replicas=128603 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] avg=65536.0000 max=66192 max_rank=5 min=64843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=193334 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32776, 65577, 32883, 65518, 131042, 98152, 32781] avg=65536.0000 max=131042 max_rank=5 min=32776 min_rank=1 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[24, 32831, 98265, 98293, 131032, 32743, 65551, 65549] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] avg=65536.0000 max=66192 max_rank=5 min=64843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=193334 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[24, 32831, 98265, 98293, 131032, 32743, 65551, 65549] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[64897, 66192, 64972, 65447, 65538, 65532, 66192, 65518] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.327132 moved_to_replicas=128603 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[24, 32831, 98265, 98293, 131032, 32743, 65551, 65549] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] avg=65536.0000 max=66192 max_rank=5 min=64843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=193334 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] avg=65536.0000 max=66192 max_rank=2 min=64823 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257945 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32776, 65577, 32883, 65518, 131042, 98152, 32781] avg=65536.0000 max=131042 max_rank=5 min=32776 min_rank=1 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[24, 32831, 98265, 98293, 131032, 32743, 65551, 65549] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] avg=65536.0000 max=66192 max_rank=2 min=64823 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257945 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] avg=65536.0000 max=66192 max_rank=2 min=64823 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257945 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[24, 32831, 98265, 98293, 131032, 32743, 65551, 65549] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32776, 65577, 32883, 65518, 131042, 98152, 32781] avg=65536.0000 max=131042 max_rank=5 min=32776 min_rank=1 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] avg=65536.0000 max=66192 max_rank=5 min=64843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=193334 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] avg=65536.0000 max=66192 max_rank=2 min=64823 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257945 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[24, 32831, 98265, 98293, 131032, 32743, 65551, 65549] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[32783, 98286, 65552, 32799, 65559, 65519, 27, 163763] avg=65536.0000 max=163763 max_rank=7 min=27 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] avg=65536.0000 max=66192 max_rank=2 min=64823 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257945 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[32783, 98286, 65552, 32799, 65559, 65519, 27, 163763] avg=65536.0000 max=163763 max_rank=7 min=27 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65536, 65577, 64843, 65518, 66192, 66192, 64871] avg=65536.0000 max=66192 max_rank=5 min=64843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=193334 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[32783, 98286, 65552, 32799, 65559, 65519, 27, 163763] avg=65536.0000 max=163763 max_rank=7 min=27 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] avg=65536.0000 max=66192 max_rank=1 min=64840 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] avg=65536.0000 max=66192 max_rank=2 min=64823 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257945 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[24, 32831, 98265, 98293, 131032, 32743, 65551, 65549] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[32783, 98286, 65552, 32799, 65559, 65519, 27, 163763] avg=65536.0000 max=163763 max_rank=7 min=27 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] avg=65536.0000 max=66192 max_rank=1 min=64840 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] avg=65536.0000 max=66192 max_rank=1 min=64840 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[32783, 98286, 65552, 32799, 65559, 65519, 27, 163763] avg=65536.0000 max=163763 max_rank=7 min=27 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[24, 32831, 98265, 98293, 131032, 32743, 65551, 65549] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] avg=65536.0000 max=66192 max_rank=2 min=64823 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257945 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] avg=65536.0000 max=66192 max_rank=1 min=64840 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[131012, 57438, 98096, 32775, 65550, 65578, 38475, 35364] avg=65536.0000 max=131012 max_rank=0 min=32775 min_rank=3 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[32783, 98286, 65552, 32799, 65559, 65519, 27, 163763] avg=65536.0000 max=163763 max_rank=7 min=27 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] avg=65536.0000 max=66192 max_rank=1 min=64840 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[131012, 57438, 98096, 32775, 65550, 65578, 38475, 35364] avg=65536.0000 max=131012 max_rank=0 min=32775 min_rank=3 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[64885, 64904, 66192, 66192, 66192, 64823, 65551, 65549] avg=65536.0000 max=66192 max_rank=2 min=64823 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257945 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[131012, 57438, 98096, 32775, 65550, 65578, 38475, 35364] avg=65536.0000 max=131012 max_rank=0 min=32775 min_rank=3 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] avg=65536.0000 max=66192 max_rank=0 min=62860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64820 max_load_reduction_pct=0.494764 moved_to_replicas=193086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] avg=65536.0000 max=66192 max_rank=1 min=64840 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[131012, 57438, 98096, 32775, 65550, 65578, 38475, 35364] avg=65536.0000 max=131012 max_rank=0 min=32775 min_rank=3 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[32783, 98286, 65552, 32799, 65559, 65519, 27, 163763] avg=65536.0000 max=163763 max_rank=7 min=27 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] avg=65536.0000 max=66192 max_rank=0 min=62860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64820 max_load_reduction_pct=0.494764 moved_to_replicas=193086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] avg=65536.0000 max=66192 max_rank=0 min=62860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64820 max_load_reduction_pct=0.494764 moved_to_replicas=193086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[131012, 57438, 98096, 32775, 65550, 65578, 38475, 35364] avg=65536.0000 max=131012 max_rank=0 min=32775 min_rank=3 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[32783, 98286, 65552, 32799, 65559, 65519, 27, 163763] avg=65536.0000 max=163763 max_rank=7 min=27 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] avg=65536.0000 max=66192 max_rank=0 min=62860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64820 max_load_reduction_pct=0.494764 moved_to_replicas=193086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] avg=65536.0000 max=66192 max_rank=1 min=64840 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[76492, 152, 87193, 98292, 65557, 32808, 131004, 32790] avg=65536.0000 max=131004 max_rank=6 min=152 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[131012, 57438, 98096, 32775, 65550, 65578, 38475, 35364] avg=65536.0000 max=131012 max_rank=0 min=32775 min_rank=3 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] avg=65536.0000 max=66192 max_rank=0 min=62860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64820 max_load_reduction_pct=0.494764 moved_to_replicas=193086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[76492, 152, 87193, 98292, 65557, 32808, 131004, 32790] avg=65536.0000 max=131004 max_rank=6 min=152 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[65541, 66192, 65552, 64893, 65559, 65519, 64840, 66192] avg=65536.0000 max=66192 max_rank=1 min=64840 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[76492, 152, 87193, 98292, 65557, 32808, 131004, 32790] avg=65536.0000 max=131004 max_rank=6 min=152 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] avg=65536.0000 max=66192 max_rank=2 min=64215 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=256212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[76492, 152, 87193, 98292, 65557, 32808, 131004, 32790] avg=65536.0000 max=131004 max_rank=6 min=152 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] avg=65536.0000 max=66192 max_rank=2 min=64215 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=256212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] avg=65536.0000 max=66192 max_rank=0 min=62860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64820 max_load_reduction_pct=0.494764 moved_to_replicas=193086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[131012, 57438, 98096, 32775, 65550, 65578, 38475, 35364] avg=65536.0000 max=131012 max_rank=0 min=32775 min_rank=3 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] avg=65536.0000 max=66192 max_rank=2 min=64215 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=256212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[76492, 152, 87193, 98292, 65557, 32808, 131004, 32790] avg=65536.0000 max=131004 max_rank=6 min=152 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[131012, 57438, 98096, 32775, 65550, 65578, 38475, 35364] avg=65536.0000 max=131012 max_rank=0 min=32775 min_rank=3 imbalance_ratio=1.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] avg=65536.0000 max=66192 max_rank=2 min=64215 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=256212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 162128, 67151, 32772, 32778, 65540] avg=65536.0000 max=162128 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.473877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] avg=65536.0000 max=66192 max_rank=0 min=62860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64820 max_load_reduction_pct=0.494764 moved_to_replicas=193086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] avg=65536.0000 max=66192 max_rank=2 min=64215 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=256212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 162128, 67151, 32772, 32778, 65540] avg=65536.0000 max=162128 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.473877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[76492, 152, 87193, 98292, 65557, 32808, 131004, 32790] avg=65536.0000 max=131004 max_rank=6 min=152 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62860, 66192, 65533, 65550, 65578, 66191, 66192] avg=65536.0000 max=66192 max_rank=0 min=62860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64820 max_load_reduction_pct=0.494764 moved_to_replicas=193086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 162128, 67151, 32772, 32778, 65540] avg=65536.0000 max=162128 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.473877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] avg=65536.0000 max=66192 max_rank=2 min=64155 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95936 max_load_reduction_pct=0.591730 moved_to_replicas=323354 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 162128, 67151, 32772, 32778, 65540] avg=65536.0000 max=162128 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.473877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] avg=65536.0000 max=66192 max_rank=2 min=64155 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95936 max_load_reduction_pct=0.591730 moved_to_replicas=323354 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[76492, 152, 87193, 98292, 65557, 32808, 131004, 32790] avg=65536.0000 max=131004 max_rank=6 min=152 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 162128, 67151, 32772, 32778, 65540] avg=65536.0000 max=162128 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.473877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] avg=65536.0000 max=66192 max_rank=2 min=64215 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=256212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] avg=65536.0000 max=66192 max_rank=2 min=64155 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95936 max_load_reduction_pct=0.591730 moved_to_replicas=323354 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] avg=65536.0000 max=66192 max_rank=2 min=64155 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95936 max_load_reduction_pct=0.591730 moved_to_replicas=323354 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[76492, 152, 87193, 98292, 65557, 32808, 131004, 32790] avg=65536.0000 max=131004 max_rank=6 min=152 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] avg=65536.0000 max=66192 max_rank=2 min=64215 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=256212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] avg=65536.0000 max=66192 max_rank=2 min=64155 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95936 max_load_reduction_pct=0.591730 moved_to_replicas=323354 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[66190, 64215, 66192, 66192, 65557, 64908, 66192, 64842] avg=65536.0000 max=66192 max_rank=2 min=64215 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=256212 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 162128, 67151, 32772, 32778, 65540] avg=65536.0000 max=162128 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.473877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 162128, 67151, 32772, 32778, 65540] avg=65536.0000 max=162128 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.473877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] avg=65536.0000 max=66192 max_rank=2 min=64155 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95936 max_load_reduction_pct=0.591730 moved_to_replicas=323354 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 162128, 67151, 32772, 32778, 65540] avg=65536.0000 max=162128 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.473877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] avg=65536.0000 max=66192 max_rank=2 min=64155 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95936 max_load_reduction_pct=0.591730 moved_to_replicas=323354 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[64910, 65576, 66192, 66192, 66192, 65531, 64155, 65540] avg=65536.0000 max=66192 max_rank=2 min=64155 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=95936 max_load_reduction_pct=0.591730 moved_to_replicas=323354 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=2088454 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=119424 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=2102496 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=103680 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=2093572 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=114944 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=2103748 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=104576 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=2106065 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=104192 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=2081251 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=126208 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=2101904 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=108160 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=2099726 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=108672 peak_demand=66193 required_capacity=1.010025 + [2026-08-30 05:15:26] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 532.6 | throughput per GPU (TFLOP/s/GPU): 625.9 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.004417E+01 | loss scale: 1.0 | grad norm: 7.317 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=69632 expected_recv=129246 expected_overflow=0 exact_recv_per_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=69632 expected_recv=130135 expected_overflow=0 exact_recv_per_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=69632 expected_recv=129246 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=69632 expected_recv=127278 expected_overflow=0 exact_recv_per_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=69632 expected_recv=130135 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=69632 expected_recv=127278 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=128640 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=69632 expected_recv=131083 expected_overflow=0 exact_recv_per_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=69632 expected_recv=131213 expected_overflow=0 exact_recv_per_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=69632 expected_recv=131083 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=69632 expected_recv=128827 expected_overflow=0 exact_recv_per_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=69632 expected_recv=131213 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=69632 expected_recv=131052 expected_overflow=0 exact_recv_per_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=69632 expected_recv=129251 expected_overflow=0 exact_recv_per_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=69632 expected_recv=128827 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=69632 expected_recv=131052 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=69632 expected_recv=129251 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=69632 expected_recv=130992 expected_overflow=0 exact_recv_per_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=69632 expected_recv=129735 expected_overflow=0 exact_recv_per_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=69632 expected_recv=128298 expected_overflow=0 exact_recv_per_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=69632 expected_recv=131105 expected_overflow=0 exact_recv_per_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=69632 expected_recv=130992 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=69632 expected_recv=129735 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=69632 expected_recv=131293 expected_overflow=0 exact_recv_per_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=69632 expected_recv=128298 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=69632 expected_recv=131105 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=69632 expected_recv=131293 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=129408 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=69632 expected_recv=130897 expected_overflow=0 exact_recv_per_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=69632 expected_recv=130225 expected_overflow=0 exact_recv_per_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=69632 expected_recv=130214 expected_overflow=0 exact_recv_per_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=69632 expected_recv=129920 expected_overflow=0 exact_recv_per_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=69632 expected_recv=130897 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=69632 expected_recv=130225 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=69632 expected_recv=130168 expected_overflow=0 exact_recv_per_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=69632 expected_recv=130214 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=69632 expected_recv=129920 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=69632 expected_recv=130168 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=69632 expected_recv=130724 expected_overflow=0 exact_recv_per_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=69632 expected_recv=130999 expected_overflow=0 exact_recv_per_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=69632 expected_recv=131023 expected_overflow=0 exact_recv_per_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=69632 expected_recv=130724 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=69632 expected_recv=130999 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=69632 expected_recv=131057 expected_overflow=0 exact_recv_per_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=69632 expected_recv=127619 expected_overflow=0 exact_recv_per_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=69632 expected_recv=131023 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=69632 expected_recv=131057 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=69632 expected_recv=127619 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=128384 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=69632 expected_recv=131139 expected_overflow=0 exact_recv_per_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=69632 expected_recv=131139 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=69632 expected_recv=129752 expected_overflow=0 exact_recv_per_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=69632 expected_recv=129777 expected_overflow=0 exact_recv_per_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=69632 expected_recv=132069 expected_overflow=0 exact_recv_per_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=69632 expected_recv=129752 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=69632 expected_recv=128687 expected_overflow=0 exact_recv_per_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=69632 expected_recv=129777 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=69632 expected_recv=132069 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=69632 expected_recv=128687 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=69632 expected_recv=129619 expected_overflow=0 exact_recv_per_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=129280 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=69632 expected_recv=129619 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=69632 expected_recv=129672 expected_overflow=0 exact_recv_per_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=69632 expected_recv=130892 expected_overflow=0 exact_recv_per_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=69632 expected_recv=130188 expected_overflow=0 exact_recv_per_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=69632 expected_recv=129672 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=69632 expected_recv=130892 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=69632 expected_recv=130188 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=69632 expected_recv=131053 expected_overflow=0 exact_recv_per_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=69632 expected_recv=131053 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=69632 expected_recv=131117 expected_overflow=0 exact_recv_per_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=69632 expected_recv=131139 expected_overflow=0 exact_recv_per_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=69632 expected_recv=131117 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=69632 expected_recv=131073 expected_overflow=0 exact_recv_per_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=69632 expected_recv=131139 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=69632 expected_recv=130138 expected_overflow=0 exact_recv_per_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=69632 expected_recv=129662 expected_overflow=0 exact_recv_per_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=69632 expected_recv=131073 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=69632 expected_recv=130138 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=69632 expected_recv=130679 expected_overflow=0 exact_recv_per_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=69632 expected_recv=129662 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=69632 expected_recv=130679 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=69632 expected_recv=129081 expected_overflow=0 exact_recv_per_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=69632 expected_recv=129081 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=69632 expected_recv=131089 expected_overflow=0 exact_recv_per_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=69632 expected_recv=129058 expected_overflow=0 exact_recv_per_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=69632 expected_recv=131089 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=69632 expected_recv=131103 expected_overflow=0 exact_recv_per_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=69632 expected_recv=129925 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=69632 expected_recv=131093 expected_overflow=0 exact_recv_per_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=69632 expected_recv=129058 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=69632 expected_recv=131103 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=69632 expected_recv=129925 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=69632 expected_recv=131093 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=69632 expected_recv=129728 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=69632 expected_recv=129934 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=69632 expected_recv=131042 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=69632 expected_recv=130795 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=69632 expected_recv=129728 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=69632 expected_recv=129934 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=69632 expected_recv=131042 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=69632 expected_recv=130795 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=69632 expected_recv=131071 expected_overflow=0 exact_recv_per_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=69632 expected_recv=131071 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=69632 expected_recv=127920 expected_overflow=0 exact_recv_per_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=69632 expected_recv=128094 expected_overflow=0 exact_recv_per_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=69632 expected_recv=131951 expected_overflow=0 exact_recv_per_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=69632 expected_recv=127920 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=69632 expected_recv=128094 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=69632 expected_recv=131951 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=128640 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=128640 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=69632 expected_recv=128628 expected_overflow=0 exact_recv_per_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=69632 expected_recv=130953 expected_overflow=0 exact_recv_per_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=69632 expected_recv=128628 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=69632 expected_recv=130953 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=129664 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=69632 expected_recv=130502 expected_overflow=0 exact_recv_per_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=69632 expected_recv=129715 expected_overflow=0 exact_recv_per_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=69632 expected_recv=131626 expected_overflow=0 exact_recv_per_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=69632 expected_recv=131251 expected_overflow=0 exact_recv_per_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=69632 expected_recv=130502 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=69632 expected_recv=129715 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=69632 expected_recv=131626 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=69632 expected_recv=131251 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=69632 expected_recv=131194 expected_overflow=0 exact_recv_per_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=69632 expected_recv=131194 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=69632 expected_recv=126882 expected_overflow=0 exact_recv_per_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=69632 expected_recv=131116 expected_overflow=0 exact_recv_per_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=69632 expected_recv=131092 expected_overflow=0 exact_recv_per_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=69632 expected_recv=132272 expected_overflow=0 exact_recv_per_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=69632 expected_recv=129696 expected_overflow=0 exact_recv_per_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=69632 expected_recv=126882 actual_valid_recv=127488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=69632 expected_recv=131116 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=69632 expected_recv=131092 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=69632 expected_recv=132272 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=127488 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=69632 expected_recv=129696 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=69632 expected_recv=129530 expected_overflow=0 exact_recv_per_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=69632 expected_recv=129530 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=69632 expected_recv=129074 expected_overflow=0 exact_recv_per_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=69632 expected_recv=130739 expected_overflow=0 exact_recv_per_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=69632 expected_recv=131083 expected_overflow=0 exact_recv_per_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=69632 expected_recv=129074 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=69632 expected_recv=130739 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130048 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=69632 expected_recv=131083 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=69632 expected_recv=131073 expected_overflow=0 exact_recv_per_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=69632 expected_recv=130097 expected_overflow=0 exact_recv_per_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=69632 expected_recv=131073 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=69632 expected_recv=130097 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=69632 expected_recv=131051 expected_overflow=0 exact_recv_per_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=69632 expected_recv=131051 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=69632 expected_recv=129934 expected_overflow=0 exact_recv_per_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=69632 expected_recv=130570 expected_overflow=0 exact_recv_per_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=69632 expected_recv=131005 expected_overflow=0 exact_recv_per_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=69632 expected_recv=129934 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=69632 expected_recv=130570 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=69632 expected_recv=131005 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=69632 expected_recv=130960 expected_overflow=0 exact_recv_per_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=69632 expected_recv=130960 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=69632 expected_recv=129586 expected_overflow=0 exact_recv_per_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=69632 expected_recv=131069 expected_overflow=0 exact_recv_per_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=69632 expected_recv=129586 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=69632 expected_recv=131069 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=69632 expected_recv=128804 expected_overflow=0 exact_recv_per_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[40191, 57788, 53548, 88915, 59853, 77357, 57583, 89053] avg=65536.0000 max=89053 max_rank=7 min=40191 min_rank=0 imbalance_ratio=1.358841 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=69632 expected_recv=128804 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[40191, 57788, 53548, 88915, 59853, 77357, 57583, 89053] avg=65536.0000 max=89053 max_rank=7 min=40191 min_rank=0 imbalance_ratio=1.358841 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=129792 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[40191, 57788, 53548, 88915, 59853, 77357, 57583, 89053] avg=65536.0000 max=89053 max_rank=7 min=40191 min_rank=0 imbalance_ratio=1.358841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63052 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=22860 max_load_reduction_pct=0.256701 moved_to_replicas=111852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63052 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=22860 max_load_reduction_pct=0.256701 moved_to_replicas=111852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[40191, 57788, 53548, 88915, 59853, 77357, 57583, 89053] avg=65536.0000 max=89053 max_rank=7 min=40191 min_rank=0 imbalance_ratio=1.358841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[40191, 57788, 53548, 88915, 59853, 77357, 57583, 89053] avg=65536.0000 max=89053 max_rank=7 min=40191 min_rank=0 imbalance_ratio=1.358841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[40191, 57788, 53548, 88915, 59853, 77357, 57583, 89053] avg=65536.0000 max=89053 max_rank=7 min=40191 min_rank=0 imbalance_ratio=1.358841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63052 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=22860 max_load_reduction_pct=0.256701 moved_to_replicas=111852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[40191, 57788, 53548, 88915, 59853, 77357, 57583, 89053] avg=65536.0000 max=89053 max_rank=7 min=40191 min_rank=0 imbalance_ratio=1.358841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[40191, 57788, 53548, 88915, 59853, 77357, 57583, 89053] avg=65536.0000 max=89053 max_rank=7 min=40191 min_rank=0 imbalance_ratio=1.358841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32892, 40526, 36, 163353, 45313, 32823, 65515, 143830] avg=65536.0000 max=163353 max_rank=3 min=36 min_rank=2 imbalance_ratio=2.492569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63052 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=22860 max_load_reduction_pct=0.256701 moved_to_replicas=111852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63052 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=22860 max_load_reduction_pct=0.256701 moved_to_replicas=111852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63052 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=22860 max_load_reduction_pct=0.256701 moved_to_replicas=111852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32892, 40526, 36, 163353, 45313, 32823, 65515, 143830] avg=65536.0000 max=163353 max_rank=3 min=36 min_rank=2 imbalance_ratio=2.492569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63052 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=22860 max_load_reduction_pct=0.256701 moved_to_replicas=111852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[63052, 66192, 66191, 66192, 64084, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63052 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=22860 max_load_reduction_pct=0.256701 moved_to_replicas=111852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32892, 40526, 36, 163353, 45313, 32823, 65515, 143830] avg=65536.0000 max=163353 max_rank=3 min=36 min_rank=2 imbalance_ratio=2.492569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] avg=65536.0000 max=66192 max_rank=3 min=64493 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97161 max_load_reduction_pct=0.594792 moved_to_replicas=345379 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] avg=65536.0000 max=66192 max_rank=3 min=64493 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97161 max_load_reduction_pct=0.594792 moved_to_replicas=345379 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32892, 40526, 36, 163353, 45313, 32823, 65515, 143830] avg=65536.0000 max=163353 max_rank=3 min=36 min_rank=2 imbalance_ratio=2.492569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32892, 40526, 36, 163353, 45313, 32823, 65515, 143830] avg=65536.0000 max=163353 max_rank=3 min=36 min_rank=2 imbalance_ratio=2.492569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32892, 40526, 36, 163353, 45313, 32823, 65515, 143830] avg=65536.0000 max=163353 max_rank=3 min=36 min_rank=2 imbalance_ratio=2.492569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] avg=65536.0000 max=66192 max_rank=3 min=64493 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97161 max_load_reduction_pct=0.594792 moved_to_replicas=345379 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32892, 40526, 36, 163353, 45313, 32823, 65515, 143830] avg=65536.0000 max=163353 max_rank=3 min=36 min_rank=2 imbalance_ratio=2.492569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32777, 98273, 50777, 81138, 65701, 98288, 65561, 31773] avg=65536.0000 max=98288 max_rank=5 min=31773 min_rank=7 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32892, 40526, 36, 163353, 45313, 32823, 65515, 143830] avg=65536.0000 max=163353 max_rank=3 min=36 min_rank=2 imbalance_ratio=2.492569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] avg=65536.0000 max=66192 max_rank=3 min=64493 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97161 max_load_reduction_pct=0.594792 moved_to_replicas=345379 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] avg=65536.0000 max=66192 max_rank=3 min=64493 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97161 max_load_reduction_pct=0.594792 moved_to_replicas=345379 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] avg=65536.0000 max=66192 max_rank=3 min=64493 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97161 max_load_reduction_pct=0.594792 moved_to_replicas=345379 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32777, 98273, 50777, 81138, 65701, 98288, 65561, 31773] avg=65536.0000 max=98288 max_rank=5 min=31773 min_rank=7 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] avg=65536.0000 max=66192 max_rank=3 min=64493 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97161 max_load_reduction_pct=0.594792 moved_to_replicas=345379 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] avg=65536.0000 max=66194 max_rank=3 min=63869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32094 max_load_reduction_pct=0.326530 moved_to_replicas=159887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[65643, 66191, 65527, 66192, 64535, 64493, 65515, 66192] avg=65536.0000 max=66192 max_rank=3 min=64493 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97161 max_load_reduction_pct=0.594792 moved_to_replicas=345379 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32777, 98273, 50777, 81138, 65701, 98288, 65561, 31773] avg=65536.0000 max=98288 max_rank=5 min=31773 min_rank=7 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] avg=65536.0000 max=66194 max_rank=3 min=63869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32094 max_load_reduction_pct=0.326530 moved_to_replicas=159887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[32777, 98273, 50777, 81138, 65701, 98288, 65561, 31773] avg=65536.0000 max=98288 max_rank=5 min=31773 min_rank=7 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[32777, 98273, 50777, 81138, 65701, 98288, 65561, 31773] avg=65536.0000 max=98288 max_rank=5 min=31773 min_rank=7 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32777, 98273, 50777, 81138, 65701, 98288, 65561, 31773] avg=65536.0000 max=98288 max_rank=5 min=31773 min_rank=7 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] avg=65536.0000 max=66194 max_rank=3 min=63869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32094 max_load_reduction_pct=0.326530 moved_to_replicas=159887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[32777, 98273, 50777, 81138, 65701, 98288, 65561, 31773] avg=65536.0000 max=98288 max_rank=5 min=31773 min_rank=7 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[68, 33011, 32804, 65424, 98280, 98251, 163559, 32891] avg=65536.0000 max=163559 max_rank=6 min=68 min_rank=0 imbalance_ratio=2.495712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] avg=65536.0000 max=66194 max_rank=3 min=63869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32094 max_load_reduction_pct=0.326530 moved_to_replicas=159887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32777, 98273, 50777, 81138, 65701, 98288, 65561, 31773] avg=65536.0000 max=98288 max_rank=5 min=31773 min_rank=7 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] avg=65536.0000 max=66194 max_rank=3 min=63869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32094 max_load_reduction_pct=0.326530 moved_to_replicas=159887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] avg=65536.0000 max=66194 max_rank=3 min=63869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32094 max_load_reduction_pct=0.326530 moved_to_replicas=159887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[68, 33011, 32804, 65424, 98280, 98251, 163559, 32891] avg=65536.0000 max=163559 max_rank=6 min=68 min_rank=0 imbalance_ratio=2.495712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] avg=65536.0000 max=66194 max_rank=3 min=63869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32094 max_load_reduction_pct=0.326530 moved_to_replicas=159887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] avg=65536.0000 max=66192 max_rank=4 min=64676 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97367 max_load_reduction_pct=0.595302 moved_to_replicas=323084 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[68, 33011, 32804, 65424, 98280, 98251, 163559, 32891] avg=65536.0000 max=163559 max_rank=6 min=68 min_rank=0 imbalance_ratio=2.495712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[64858, 66192, 65721, 66194, 65701, 66192, 65561, 63869] avg=65536.0000 max=66194 max_rank=3 min=63869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32094 max_load_reduction_pct=0.326530 moved_to_replicas=159887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] avg=65536.0000 max=66192 max_rank=4 min=64676 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97367 max_load_reduction_pct=0.595302 moved_to_replicas=323084 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[68, 33011, 32804, 65424, 98280, 98251, 163559, 32891] avg=65536.0000 max=163559 max_rank=6 min=68 min_rank=0 imbalance_ratio=2.495712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[68, 33011, 32804, 65424, 98280, 98251, 163559, 32891] avg=65536.0000 max=163559 max_rank=6 min=68 min_rank=0 imbalance_ratio=2.495712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[68, 33011, 32804, 65424, 98280, 98251, 163559, 32891] avg=65536.0000 max=163559 max_rank=6 min=68 min_rank=0 imbalance_ratio=2.495712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[68, 33011, 32804, 65424, 98280, 98251, 163559, 32891] avg=65536.0000 max=163559 max_rank=6 min=68 min_rank=0 imbalance_ratio=2.495712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] avg=65536.0000 max=66192 max_rank=4 min=64676 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97367 max_load_reduction_pct=0.595302 moved_to_replicas=323084 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32783, 65527, 32505, 71808, 99916, 98249, 58000, 65500] avg=65536.0000 max=99916 max_rank=4 min=32505 min_rank=2 imbalance_ratio=1.524597 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] avg=65536.0000 max=66192 max_rank=4 min=64676 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97367 max_load_reduction_pct=0.595302 moved_to_replicas=323084 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[68, 33011, 32804, 65424, 98280, 98251, 163559, 32891] avg=65536.0000 max=163559 max_rank=6 min=68 min_rank=0 imbalance_ratio=2.495712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32783, 65527, 32505, 71808, 99916, 98249, 58000, 65500] avg=65536.0000 max=99916 max_rank=4 min=32505 min_rank=2 imbalance_ratio=1.524597 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] avg=65536.0000 max=66192 max_rank=4 min=64676 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97367 max_load_reduction_pct=0.595302 moved_to_replicas=323084 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] avg=65536.0000 max=66192 max_rank=4 min=64676 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97367 max_load_reduction_pct=0.595302 moved_to_replicas=323084 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] avg=65536.0000 max=66192 max_rank=4 min=64676 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97367 max_load_reduction_pct=0.595302 moved_to_replicas=323084 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] avg=65536.0000 max=66194 max_rank=3 min=63614 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33722 max_load_reduction_pct=0.337504 moved_to_replicas=139589 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] avg=65536.0000 max=66194 max_rank=3 min=63614 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33722 max_load_reduction_pct=0.337504 moved_to_replicas=139589 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32783, 65527, 32505, 71808, 99916, 98249, 58000, 65500] avg=65536.0000 max=99916 max_rank=4 min=32505 min_rank=2 imbalance_ratio=1.524597 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[64676, 65070, 65563, 65424, 66192, 66192, 66192, 64979] avg=65536.0000 max=66192 max_rank=4 min=64676 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97367 max_load_reduction_pct=0.595302 moved_to_replicas=323084 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32783, 65527, 32505, 71808, 99916, 98249, 58000, 65500] avg=65536.0000 max=99916 max_rank=4 min=32505 min_rank=2 imbalance_ratio=1.524597 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32783, 65527, 32505, 71808, 99916, 98249, 58000, 65500] avg=65536.0000 max=99916 max_rank=4 min=32505 min_rank=2 imbalance_ratio=1.524597 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32783, 65527, 32505, 71808, 99916, 98249, 58000, 65500] avg=65536.0000 max=99916 max_rank=4 min=32505 min_rank=2 imbalance_ratio=1.524597 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32783, 65527, 32505, 71808, 99916, 98249, 58000, 65500] avg=65536.0000 max=99916 max_rank=4 min=32505 min_rank=2 imbalance_ratio=1.524597 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34190, 97937, 65585, 98288, 98267, 64445, 32803] avg=65536.0000 max=98288 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] avg=65536.0000 max=66194 max_rank=3 min=63614 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33722 max_load_reduction_pct=0.337504 moved_to_replicas=139589 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] avg=65536.0000 max=66194 max_rank=3 min=63614 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33722 max_load_reduction_pct=0.337504 moved_to_replicas=139589 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] avg=65536.0000 max=66194 max_rank=3 min=63614 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33722 max_load_reduction_pct=0.337504 moved_to_replicas=139589 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] avg=65536.0000 max=66194 max_rank=3 min=63614 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33722 max_load_reduction_pct=0.337504 moved_to_replicas=139589 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34190, 97937, 65585, 98288, 98267, 64445, 32803] avg=65536.0000 max=98288 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] avg=65536.0000 max=66192 max_rank=2 min=64445 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32096 max_load_reduction_pct=0.326551 moved_to_replicas=191132 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] avg=65536.0000 max=66194 max_rank=3 min=63614 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33722 max_load_reduction_pct=0.337504 moved_to_replicas=139589 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32783, 65527, 32505, 71808, 99916, 98249, 58000, 65500] avg=65536.0000 max=99916 max_rank=4 min=32505 min_rank=2 imbalance_ratio=1.524597 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] avg=65536.0000 max=66192 max_rank=2 min=64445 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32096 max_load_reduction_pct=0.326551 moved_to_replicas=191132 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34190, 97937, 65585, 98288, 98267, 64445, 32803] avg=65536.0000 max=98288 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34190, 97937, 65585, 98288, 98267, 64445, 32803] avg=65536.0000 max=98288 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34190, 97937, 65585, 98288, 98267, 64445, 32803] avg=65536.0000 max=98288 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34190, 97937, 65585, 98288, 98267, 64445, 32803] avg=65536.0000 max=98288 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[65804, 65527, 65265, 66194, 66192, 66192, 63614, 65500] avg=65536.0000 max=66194 max_rank=3 min=63614 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=33722 max_load_reduction_pct=0.337504 moved_to_replicas=139589 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32794, 65097, 32728, 32773, 65523, 68769, 131240, 95364] avg=65536.0000 max=131240 max_rank=6 min=32728 min_rank=2 imbalance_ratio=2.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34190, 97937, 65585, 98288, 98267, 64445, 32803] avg=65536.0000 max=98288 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] avg=65536.0000 max=66192 max_rank=2 min=64445 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32096 max_load_reduction_pct=0.326551 moved_to_replicas=191132 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] avg=65536.0000 max=66192 max_rank=2 min=64445 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32096 max_load_reduction_pct=0.326551 moved_to_replicas=191132 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] avg=65536.0000 max=66192 max_rank=2 min=64445 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32096 max_load_reduction_pct=0.326551 moved_to_replicas=191132 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] avg=65536.0000 max=66192 max_rank=2 min=64445 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32096 max_load_reduction_pct=0.326551 moved_to_replicas=191132 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32794, 65097, 32728, 32773, 65523, 68769, 131240, 95364] avg=65536.0000 max=131240 max_rank=6 min=32728 min_rank=2 imbalance_ratio=2.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65048 max_load_reduction_pct=0.495642 moved_to_replicas=193734 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] avg=65536.0000 max=66192 max_rank=2 min=64445 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32096 max_load_reduction_pct=0.326551 moved_to_replicas=191132 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34190, 97937, 65585, 98288, 98267, 64445, 32803] avg=65536.0000 max=98288 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65048 max_load_reduction_pct=0.495642 moved_to_replicas=193734 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32794, 65097, 32728, 32773, 65523, 68769, 131240, 95364] avg=65536.0000 max=131240 max_rank=6 min=32728 min_rank=2 imbalance_ratio=2.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32794, 65097, 32728, 32773, 65523, 68769, 131240, 95364] avg=65536.0000 max=131240 max_rank=6 min=32728 min_rank=2 imbalance_ratio=2.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32794, 65097, 32728, 32773, 65523, 68769, 131240, 95364] avg=65536.0000 max=131240 max_rank=6 min=32728 min_rank=2 imbalance_ratio=2.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32794, 65097, 32728, 32773, 65523, 68769, 131240, 95364] avg=65536.0000 max=131240 max_rank=6 min=32728 min_rank=2 imbalance_ratio=2.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[64869, 65935, 66192, 65585, 66192, 66192, 64445, 64878] avg=65536.0000 max=66192 max_rank=2 min=64445 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32096 max_load_reduction_pct=0.326551 moved_to_replicas=191132 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 130993, 98465, 65302, 21, 32828, 65586] avg=65536.0000 max=130993 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32794, 65097, 32728, 32773, 65523, 68769, 131240, 95364] avg=65536.0000 max=131240 max_rank=6 min=32728 min_rank=2 imbalance_ratio=2.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65048 max_load_reduction_pct=0.495642 moved_to_replicas=193734 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65048 max_load_reduction_pct=0.495642 moved_to_replicas=193734 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 130993, 98465, 65302, 21, 32828, 65586] avg=65536.0000 max=130993 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65048 max_load_reduction_pct=0.495642 moved_to_replicas=193734 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65048 max_load_reduction_pct=0.495642 moved_to_replicas=193734 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] avg=65536.0000 max=66192 max_rank=2 min=64822 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64801 max_load_reduction_pct=0.494691 moved_to_replicas=194146 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65048 max_load_reduction_pct=0.495642 moved_to_replicas=193734 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32794, 65097, 32728, 32773, 65523, 68769, 131240, 95364] avg=65536.0000 max=131240 max_rank=6 min=32728 min_rank=2 imbalance_ratio=2.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] avg=65536.0000 max=66192 max_rank=2 min=64822 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64801 max_load_reduction_pct=0.494691 moved_to_replicas=194146 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 130993, 98465, 65302, 21, 32828, 65586] avg=65536.0000 max=130993 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 130993, 98465, 65302, 21, 32828, 65586] avg=65536.0000 max=130993 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 130993, 98465, 65302, 21, 32828, 65586] avg=65536.0000 max=130993 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 130993, 98465, 65302, 21, 32828, 65586] avg=65536.0000 max=130993 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[64543, 65097, 65484, 65065, 65523, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64543 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65048 max_load_reduction_pct=0.495642 moved_to_replicas=193734 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[45, 65538, 54, 130957, 65551, 65554, 98289, 98300] avg=65536.0000 max=130957 max_rank=3 min=45 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 130993, 98465, 65302, 21, 32828, 65586] avg=65536.0000 max=130993 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] avg=65536.0000 max=66192 max_rank=2 min=64822 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64801 max_load_reduction_pct=0.494691 moved_to_replicas=194146 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] avg=65536.0000 max=66192 max_rank=2 min=64822 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64801 max_load_reduction_pct=0.494691 moved_to_replicas=194146 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[45, 65538, 54, 130957, 65551, 65554, 98289, 98300] avg=65536.0000 max=130957 max_rank=3 min=45 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] avg=65536.0000 max=66192 max_rank=2 min=64822 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64801 max_load_reduction_pct=0.494691 moved_to_replicas=194146 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] avg=65536.0000 max=66192 max_rank=2 min=64822 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64801 max_load_reduction_pct=0.494691 moved_to_replicas=194146 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64171 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=258012 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] avg=65536.0000 max=66192 max_rank=2 min=64822 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64801 max_load_reduction_pct=0.494691 moved_to_replicas=194146 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 130993, 98465, 65302, 21, 32828, 65586] avg=65536.0000 max=130993 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64171 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=258012 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[45, 65538, 54, 130957, 65551, 65554, 98289, 98300] avg=65536.0000 max=130957 max_rank=3 min=45 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[45, 65538, 54, 130957, 65551, 65554, 98289, 98300] avg=65536.0000 max=130957 max_rank=3 min=45 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[45, 65538, 54, 130957, 65551, 65554, 98289, 98300] avg=65536.0000 max=130957 max_rank=3 min=45 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[45, 65538, 54, 130957, 65551, 65554, 98289, 98300] avg=65536.0000 max=130957 max_rank=3 min=45 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[131017, 98280, 32804, 110, 65405, 65514, 32873, 98285] avg=65536.0000 max=131017 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[65552, 65541, 66192, 66192, 65302, 64822, 65101, 65586] avg=65536.0000 max=66192 max_rank=2 min=64822 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64801 max_load_reduction_pct=0.494691 moved_to_replicas=194146 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[45, 65538, 54, 130957, 65551, 65554, 98289, 98300] avg=65536.0000 max=130957 max_rank=3 min=45 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64171 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=258012 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64171 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=258012 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[131017, 98280, 32804, 110, 65405, 65514, 32873, 98285] avg=65536.0000 max=131017 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64171 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=258012 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64171 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=258012 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] avg=65536.0000 max=66192 max_rank=0 min=64873 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.494783 moved_to_replicas=258158 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64171 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=258012 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] avg=65536.0000 max=66192 max_rank=0 min=64873 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.494783 moved_to_replicas=258158 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[45, 65538, 54, 130957, 65551, 65554, 98289, 98300] avg=65536.0000 max=130957 max_rank=3 min=45 min_rank=0 imbalance_ratio=1.998245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[131017, 98280, 32804, 110, 65405, 65514, 32873, 98285] avg=65536.0000 max=131017 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[131017, 98280, 32804, 110, 65405, 65514, 32873, 98285] avg=65536.0000 max=131017 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[131017, 98280, 32804, 110, 65405, 65514, 32873, 98285] avg=65536.0000 max=131017 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[131017, 98280, 32804, 110, 65405, 65514, 32873, 98285] avg=65536.0000 max=131017 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[106818, 32783, 32808, 34083, 120731, 66079, 98206, 32780] avg=65536.0000 max=120731 max_rank=4 min=32780 min_rank=7 imbalance_ratio=1.842209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65538, 64171, 66192, 65551, 65554, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64171 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64765 max_load_reduction_pct=0.494552 moved_to_replicas=258012 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[131017, 98280, 32804, 110, 65405, 65514, 32873, 98285] avg=65536.0000 max=131017 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] avg=65536.0000 max=66192 max_rank=0 min=64873 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.494783 moved_to_replicas=258158 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] avg=65536.0000 max=66192 max_rank=0 min=64873 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.494783 moved_to_replicas=258158 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[106818, 32783, 32808, 34083, 120731, 66079, 98206, 32780] avg=65536.0000 max=120731 max_rank=4 min=32780 min_rank=7 imbalance_ratio=1.842209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] avg=65536.0000 max=66192 max_rank=0 min=64873 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.494783 moved_to_replicas=258158 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] avg=65536.0000 max=66193 max_rank=0 min=62391 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=54538 max_load_reduction_pct=0.451732 moved_to_replicas=255440 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] avg=65536.0000 max=66192 max_rank=0 min=64873 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.494783 moved_to_replicas=258158 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] avg=65536.0000 max=66192 max_rank=0 min=64873 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.494783 moved_to_replicas=258158 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] avg=65536.0000 max=66193 max_rank=0 min=62391 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=54538 max_load_reduction_pct=0.451732 moved_to_replicas=255440 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[131017, 98280, 32804, 110, 65405, 65514, 32873, 98285] avg=65536.0000 max=131017 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[106818, 32783, 32808, 34083, 120731, 66079, 98206, 32780] avg=65536.0000 max=120731 max_rank=4 min=32780 min_rank=7 imbalance_ratio=1.842209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[106818, 32783, 32808, 34083, 120731, 66079, 98206, 32780] avg=65536.0000 max=120731 max_rank=4 min=32780 min_rank=7 imbalance_ratio=1.842209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[106818, 32783, 32808, 34083, 120731, 66079, 98206, 32780] avg=65536.0000 max=120731 max_rank=4 min=32780 min_rank=7 imbalance_ratio=1.842209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[88, 98245, 64955, 98322, 32814, 66088, 98307, 65469] avg=65536.0000 max=98322 max_rank=3 min=88 min_rank=0 imbalance_ratio=1.500275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[106818, 32783, 32808, 34083, 120731, 66079, 98206, 32780] avg=65536.0000 max=120731 max_rank=4 min=32780 min_rank=7 imbalance_ratio=1.842209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64873, 64959, 65405, 65514, 64961, 66192] avg=65536.0000 max=66192 max_rank=0 min=64873 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64825 max_load_reduction_pct=0.494783 moved_to_replicas=258158 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] avg=65536.0000 max=66193 max_rank=0 min=62391 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=54538 max_load_reduction_pct=0.451732 moved_to_replicas=255440 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[106818, 32783, 32808, 34083, 120731, 66079, 98206, 32780] avg=65536.0000 max=120731 max_rank=4 min=32780 min_rank=7 imbalance_ratio=1.842209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[88, 98245, 64955, 98322, 32814, 66088, 98307, 65469] avg=65536.0000 max=98322 max_rank=3 min=88 min_rank=0 imbalance_ratio=1.500275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] avg=65536.0000 max=66193 max_rank=0 min=62391 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=54538 max_load_reduction_pct=0.451732 moved_to_replicas=255440 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] avg=65536.0000 max=66193 max_rank=0 min=62391 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=54538 max_load_reduction_pct=0.451732 moved_to_replicas=255440 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] avg=65536.0000 max=66192 max_rank=1 min=64333 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32130 max_load_reduction_pct=0.326783 moved_to_replicas=192624 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] avg=65536.0000 max=66193 max_rank=0 min=62391 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=54538 max_load_reduction_pct=0.451732 moved_to_replicas=255440 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] avg=65536.0000 max=66193 max_rank=0 min=62391 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=54538 max_load_reduction_pct=0.451732 moved_to_replicas=255440 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] avg=65536.0000 max=66192 max_rank=1 min=64333 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32130 max_load_reduction_pct=0.326783 moved_to_replicas=192624 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[106818, 32783, 32808, 34083, 120731, 66079, 98206, 32780] avg=65536.0000 max=120731 max_rank=4 min=32780 min_rank=7 imbalance_ratio=1.842209 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[88, 98245, 64955, 98322, 32814, 66088, 98307, 65469] avg=65536.0000 max=98322 max_rank=3 min=88 min_rank=0 imbalance_ratio=1.500275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[88, 98245, 64955, 98322, 32814, 66088, 98307, 65469] avg=65536.0000 max=98322 max_rank=3 min=88 min_rank=0 imbalance_ratio=1.500275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[88, 98245, 64955, 98322, 32814, 66088, 98307, 65469] avg=65536.0000 max=98322 max_rank=3 min=88 min_rank=0 imbalance_ratio=1.500275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32880, 57975, 65580, 65677, 32802, 34174, 71607, 163593] avg=65536.0000 max=163593 max_rank=7 min=32802 min_rank=4 imbalance_ratio=2.496231 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[88, 98245, 64955, 98322, 32814, 66088, 98307, 65469] avg=65536.0000 max=98322 max_rank=3 min=88 min_rank=0 imbalance_ratio=1.500275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62391, 65511, 66193, 66193, 66079, 66192, 65536] avg=65536.0000 max=66193 max_rank=0 min=62391 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=54538 max_load_reduction_pct=0.451732 moved_to_replicas=255440 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] avg=65536.0000 max=66192 max_rank=1 min=64333 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32130 max_load_reduction_pct=0.326783 moved_to_replicas=192624 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[88, 98245, 64955, 98322, 32814, 66088, 98307, 65469] avg=65536.0000 max=98322 max_rank=3 min=88 min_rank=0 imbalance_ratio=1.500275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32880, 57975, 65580, 65677, 32802, 34174, 71607, 163593] avg=65536.0000 max=163593 max_rank=7 min=32802 min_rank=4 imbalance_ratio=2.496231 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] avg=65536.0000 max=66192 max_rank=1 min=64333 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32130 max_load_reduction_pct=0.326783 moved_to_replicas=192624 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] avg=65536.0000 max=66192 max_rank=1 min=64333 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32130 max_load_reduction_pct=0.326783 moved_to_replicas=192624 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63389 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=97400 max_load_reduction_pct=0.595380 moved_to_replicas=204240 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] avg=65536.0000 max=66192 max_rank=1 min=64333 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32130 max_load_reduction_pct=0.326783 moved_to_replicas=192624 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] avg=65536.0000 max=66192 max_rank=1 min=64333 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32130 max_load_reduction_pct=0.326783 moved_to_replicas=192624 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63389 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=97400 max_load_reduction_pct=0.595380 moved_to_replicas=204240 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32880, 57975, 65580, 65677, 32802, 34174, 71607, 163593] avg=65536.0000 max=163593 max_rank=7 min=32802 min_rank=4 imbalance_ratio=2.496231 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[88, 98245, 64955, 98322, 32814, 66088, 98307, 65469] avg=65536.0000 max=98322 max_rank=3 min=88 min_rank=0 imbalance_ratio=1.500275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32880, 57975, 65580, 65677, 32802, 34174, 71607, 163593] avg=65536.0000 max=163593 max_rank=7 min=32802 min_rank=4 imbalance_ratio=2.496231 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32880, 57975, 65580, 65677, 32802, 34174, 71607, 163593] avg=65536.0000 max=163593 max_rank=7 min=32802 min_rank=4 imbalance_ratio=2.496231 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[98176, 31941, 131116, 32798, 32836, 71024, 93817, 32580] avg=65536.0000 max=131116 max_rank=2 min=31941 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32880, 57975, 65580, 65677, 32802, 34174, 71607, 163593] avg=65536.0000 max=163593 max_rank=7 min=32802 min_rank=4 imbalance_ratio=2.496231 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63389 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=97400 max_load_reduction_pct=0.595380 moved_to_replicas=204240 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[64333, 66192, 64955, 66192, 64867, 66088, 66192, 65469] avg=65536.0000 max=66192 max_rank=1 min=64333 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32130 max_load_reduction_pct=0.326783 moved_to_replicas=192624 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63389 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=97400 max_load_reduction_pct=0.595380 moved_to_replicas=204240 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[98176, 31941, 131116, 32798, 32836, 71024, 93817, 32580] avg=65536.0000 max=131116 max_rank=2 min=31941 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32880, 57975, 65580, 65677, 32802, 34174, 71607, 163593] avg=65536.0000 max=163593 max_rank=7 min=32802 min_rank=4 imbalance_ratio=2.496231 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63389 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=97400 max_load_reduction_pct=0.595380 moved_to_replicas=204240 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] avg=65536.0000 max=66193 max_rank=6 min=64696 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=259011 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63389 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=97400 max_load_reduction_pct=0.595380 moved_to_replicas=204240 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[98176, 31941, 131116, 32798, 32836, 71024, 93817, 32580] avg=65536.0000 max=131116 max_rank=2 min=31941 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] avg=65536.0000 max=66193 max_rank=6 min=64696 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=259011 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63389 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=97400 max_load_reduction_pct=0.595380 moved_to_replicas=204240 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32880, 57975, 65580, 65677, 32802, 34174, 71607, 163593] avg=65536.0000 max=163593 max_rank=7 min=32802 min_rank=4 imbalance_ratio=2.496231 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[98176, 31941, 131116, 32798, 32836, 71024, 93817, 32580] avg=65536.0000 max=131116 max_rank=2 min=31941 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[98176, 31941, 131116, 32798, 32836, 71024, 93817, 32580] avg=65536.0000 max=131116 max_rank=2 min=31941 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[33313, 65527, 32322, 65544, 131207, 32779, 98059, 65537] avg=65536.0000 max=131207 max_rank=4 min=32322 min_rank=2 imbalance_ratio=2.002060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] avg=65536.0000 max=66193 max_rank=6 min=64696 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=259011 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[98176, 31941, 131116, 32798, 32836, 71024, 93817, 32580] avg=65536.0000 max=131116 max_rank=2 min=31941 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[65621, 63389, 65580, 65677, 65556, 66080, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63389 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=97400 max_load_reduction_pct=0.595380 moved_to_replicas=204240 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[33313, 65527, 32322, 65544, 131207, 32779, 98059, 65537] avg=65536.0000 max=131207 max_rank=4 min=32322 min_rank=2 imbalance_ratio=2.002060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] avg=65536.0000 max=66193 max_rank=6 min=64696 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=259011 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[98176, 31941, 131116, 32798, 32836, 71024, 93817, 32580] avg=65536.0000 max=131116 max_rank=2 min=31941 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] avg=65536.0000 max=66193 max_rank=6 min=64696 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=259011 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=65037 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65015 max_load_reduction_pct=0.495515 moved_to_replicas=193728 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] avg=65536.0000 max=66193 max_rank=6 min=64696 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=259011 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=65037 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65015 max_load_reduction_pct=0.495515 moved_to_replicas=193728 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[33313, 65527, 32322, 65544, 131207, 32779, 98059, 65537] avg=65536.0000 max=131207 max_rank=4 min=32322 min_rank=2 imbalance_ratio=2.002060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] avg=65536.0000 max=66193 max_rank=6 min=64696 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=259011 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[98176, 31941, 131116, 32798, 32836, 71024, 93817, 32580] avg=65536.0000 max=131116 max_rank=2 min=31941 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[33313, 65527, 32322, 65544, 131207, 32779, 98059, 65537] avg=65536.0000 max=131207 max_rank=4 min=32322 min_rank=2 imbalance_ratio=2.002060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[33313, 65527, 32322, 65544, 131207, 32779, 98059, 65537] avg=65536.0000 max=131207 max_rank=4 min=32322 min_rank=2 imbalance_ratio=2.002060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[65511, 98274, 98494, 65485, 65529, 124, 32859, 98012] avg=65536.0000 max=98494 max_rank=2 min=124 min_rank=5 imbalance_ratio=1.502899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=65037 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65015 max_load_reduction_pct=0.495515 moved_to_replicas=193728 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[33313, 65527, 32322, 65544, 131207, 32779, 98059, 65537] avg=65536.0000 max=131207 max_rank=4 min=32322 min_rank=2 imbalance_ratio=2.002060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64696, 66192, 64782, 65293, 66191, 66193, 64749] avg=65536.0000 max=66193 max_rank=6 min=64696 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=259011 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[65511, 98274, 98494, 65485, 65529, 124, 32859, 98012] avg=65536.0000 max=98494 max_rank=2 min=124 min_rank=5 imbalance_ratio=1.502899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=65037 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65015 max_load_reduction_pct=0.495515 moved_to_replicas=193728 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=65037 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65015 max_load_reduction_pct=0.495515 moved_to_replicas=193728 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[33313, 65527, 32322, 65544, 131207, 32779, 98059, 65537] avg=65536.0000 max=131207 max_rank=4 min=32322 min_rank=2 imbalance_ratio=2.002060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] avg=65536.0000 max=66192 max_rank=1 min=64508 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.327959 moved_to_replicas=192529 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=65037 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65015 max_load_reduction_pct=0.495515 moved_to_replicas=193728 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] avg=65536.0000 max=66192 max_rank=1 min=64508 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.327959 moved_to_replicas=192529 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[65511, 98274, 98494, 65485, 65529, 124, 32859, 98012] avg=65536.0000 max=98494 max_rank=2 min=124 min_rank=5 imbalance_ratio=1.502899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=65037 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65015 max_load_reduction_pct=0.495515 moved_to_replicas=193728 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[33313, 65527, 32322, 65544, 131207, 32779, 98059, 65537] avg=65536.0000 max=131207 max_rank=4 min=32322 min_rank=2 imbalance_ratio=2.002060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[65511, 98274, 98494, 65485, 65529, 124, 32859, 98012] avg=65536.0000 max=98494 max_rank=2 min=124 min_rank=5 imbalance_ratio=1.502899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[65511, 98274, 98494, 65485, 65529, 124, 32859, 98012] avg=65536.0000 max=98494 max_rank=2 min=124 min_rank=5 imbalance_ratio=1.502899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] avg=65536.0000 max=66192 max_rank=1 min=64508 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.327959 moved_to_replicas=192529 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[65511, 98274, 98494, 65485, 65529, 124, 32859, 98012] avg=65536.0000 max=98494 max_rank=2 min=124 min_rank=5 imbalance_ratio=1.502899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[65180, 65527, 65079, 65544, 66192, 65037, 66192, 65537] avg=65536.0000 max=66192 max_rank=4 min=65037 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65015 max_load_reduction_pct=0.495515 moved_to_replicas=193728 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] avg=65536.0000 max=66192 max_rank=1 min=64508 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.327959 moved_to_replicas=192529 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] avg=65536.0000 max=66192 max_rank=1 min=64508 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.327959 moved_to_replicas=192529 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[65511, 98274, 98494, 65485, 65529, 124, 32859, 98012] avg=65536.0000 max=98494 max_rank=2 min=124 min_rank=5 imbalance_ratio=1.502899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] avg=65536.0000 max=66192 max_rank=1 min=64508 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.327959 moved_to_replicas=192529 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] avg=65536.0000 max=66192 max_rank=1 min=64508 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.327959 moved_to_replicas=192529 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[65511, 98274, 98494, 65485, 65529, 124, 32859, 98012] avg=65536.0000 max=98494 max_rank=2 min=124 min_rank=5 imbalance_ratio=1.502899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[65511, 66192, 66192, 65485, 65529, 64508, 64679, 66192] avg=65536.0000 max=66192 max_rank=1 min=64508 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.327959 moved_to_replicas=192529 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=2103690 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=109696 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=2099471 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=114048 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=2095979 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=117760 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=2097331 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=116096 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=2094873 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=120832 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=2089061 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=124032 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=2094267 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=119680 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=2102544 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=112640 peak_demand=66192 required_capacity=1.010010 + [2026-08-30 05:15:26] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 523.1 | throughput per GPU (TFLOP/s/GPU): 637.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.519349E+00 | loss scale: 1.0 | grad norm: 5.761 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=69632 expected_recv=131622 expected_overflow=0 exact_recv_per_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=69632 expected_recv=128312 expected_overflow=0 exact_recv_per_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=69632 expected_recv=131622 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=69632 expected_recv=128312 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=69632 expected_recv=126721 expected_overflow=0 exact_recv_per_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=129536 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=69632 expected_recv=126721 actual_valid_recv=127872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=127872 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=69632 expected_recv=131191 expected_overflow=0 exact_recv_per_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=69632 expected_recv=130917 expected_overflow=0 exact_recv_per_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=69632 expected_recv=131191 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=69632 expected_recv=130917 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=69632 expected_recv=131031 expected_overflow=0 exact_recv_per_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=69632 expected_recv=125902 expected_overflow=0 exact_recv_per_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=69632 expected_recv=131031 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=69632 expected_recv=125902 actual_valid_recv=127488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=127488 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=69632 expected_recv=130754 expected_overflow=0 exact_recv_per_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=69632 expected_recv=128595 expected_overflow=0 exact_recv_per_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=69632 expected_recv=130754 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=69632 expected_recv=128595 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=69632 expected_recv=131276 expected_overflow=0 exact_recv_per_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=69632 expected_recv=129696 expected_overflow=0 exact_recv_per_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=69632 expected_recv=131104 expected_overflow=0 exact_recv_per_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=69632 expected_recv=131276 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=69632 expected_recv=129696 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=69632 expected_recv=131104 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=69632 expected_recv=131126 expected_overflow=0 exact_recv_per_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=69632 expected_recv=130999 expected_overflow=0 exact_recv_per_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=69632 expected_recv=130987 expected_overflow=0 exact_recv_per_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=69632 expected_recv=131126 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=69632 expected_recv=130999 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=69632 expected_recv=130987 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=69632 expected_recv=131060 expected_overflow=0 exact_recv_per_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=69632 expected_recv=129931 expected_overflow=0 exact_recv_per_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=69632 expected_recv=130839 expected_overflow=0 exact_recv_per_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=69632 expected_recv=131250 expected_overflow=0 exact_recv_per_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=69632 expected_recv=131060 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=69632 expected_recv=129931 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=69632 expected_recv=130839 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=69632 expected_recv=131250 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132389 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=69632 expected_recv=131060 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132381 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132389 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=69632 expected_recv=131060 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=69632 expected_recv=123212 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132379 expected_overflow=0 exact_recv_per_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=69632 expected_recv=123212 actual_valid_recv=123904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=69632 expected_recv=132379 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=123904 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=69632 expected_recv=127180 expected_overflow=0 exact_recv_per_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=69632 expected_recv=130126 expected_overflow=0 exact_recv_per_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=69632 expected_recv=127180 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=128128 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=69632 expected_recv=130126 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=69632 expected_recv=129649 expected_overflow=0 exact_recv_per_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=69632 expected_recv=129649 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=69632 expected_recv=132085 expected_overflow=0 exact_recv_per_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=69632 expected_recv=132085 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=69632 expected_recv=131071 expected_overflow=0 exact_recv_per_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=69632 expected_recv=131071 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=69632 expected_recv=132325 expected_overflow=0 exact_recv_per_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=69632 expected_recv=131078 expected_overflow=0 exact_recv_per_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=69632 expected_recv=132325 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=69632 expected_recv=131078 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=69632 expected_recv=130252 expected_overflow=0 exact_recv_per_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=69632 expected_recv=129064 expected_overflow=0 exact_recv_per_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=69632 expected_recv=130958 expected_overflow=0 exact_recv_per_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=69632 expected_recv=130252 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=69632 expected_recv=131444 expected_overflow=0 exact_recv_per_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=69632 expected_recv=129064 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=69632 expected_recv=130958 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=69632 expected_recv=131444 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=69632 expected_recv=129834 expected_overflow=0 exact_recv_per_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=69632 expected_recv=129834 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=69632 expected_recv=131066 expected_overflow=0 exact_recv_per_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=69632 expected_recv=131066 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=69632 expected_recv=128623 expected_overflow=0 exact_recv_per_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=69632 expected_recv=128623 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=69632 expected_recv=130819 expected_overflow=0 exact_recv_per_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=69632 expected_recv=131084 expected_overflow=0 exact_recv_per_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=130048 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=69632 expected_recv=126327 expected_overflow=0 exact_recv_per_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=69632 expected_recv=130819 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=69632 expected_recv=131084 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=69632 expected_recv=131069 expected_overflow=0 exact_recv_per_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=69632 expected_recv=126327 actual_valid_recv=127232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=69632 expected_recv=131069 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=127232 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=69632 expected_recv=131770 expected_overflow=0 exact_recv_per_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=69632 expected_recv=131770 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=69632 expected_recv=131100 expected_overflow=0 exact_recv_per_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=69632 expected_recv=131100 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=69632 expected_recv=131159 expected_overflow=0 exact_recv_per_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=69632 expected_recv=131091 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=69632 expected_recv=131082 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=69632 expected_recv=127260 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=69632 expected_recv=131159 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=69632 expected_recv=131091 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=69632 expected_recv=131082 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=69632 expected_recv=127260 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=128768 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=69632 expected_recv=131251 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=69632 expected_recv=131849 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=69632 expected_recv=129435 expected_overflow=0 exact_recv_per_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=69632 expected_recv=131251 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=69632 expected_recv=127781 expected_overflow=0 exact_recv_per_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=69632 expected_recv=131276 expected_overflow=0 exact_recv_per_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=69632 expected_recv=131849 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=69632 expected_recv=129435 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=69632 expected_recv=127781 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=69632 expected_recv=131276 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130560 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=129280 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=69632 expected_recv=129438 expected_overflow=0 exact_recv_per_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=69632 expected_recv=129438 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=69632 expected_recv=131086 expected_overflow=0 exact_recv_per_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=69632 expected_recv=131086 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=69632 expected_recv=131144 expected_overflow=0 exact_recv_per_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=69632 expected_recv=131144 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=69632 expected_recv=130800 expected_overflow=0 exact_recv_per_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=69632 expected_recv=130530 expected_overflow=0 exact_recv_per_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=69632 expected_recv=130800 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=69632 expected_recv=127862 expected_overflow=0 exact_recv_per_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=69632 expected_recv=130530 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=69632 expected_recv=127293 expected_overflow=0 exact_recv_per_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=69632 expected_recv=131104 expected_overflow=0 exact_recv_per_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=69632 expected_recv=127862 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=69632 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=69632 expected_recv=127293 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=69632 expected_recv=131104 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=129408 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=69632 expected_recv=132387 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=128640 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=69632 expected_recv=130111 expected_overflow=0 exact_recv_per_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=69632 expected_recv=130111 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=69632 expected_recv=131054 expected_overflow=0 exact_recv_per_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=69632 expected_recv=132232 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=69632 expected_recv=131054 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=69632 expected_recv=131335 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=69632 expected_recv=131860 expected_overflow=0 exact_recv_per_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=69632 expected_recv=132232 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=69632 expected_recv=131335 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=69632 expected_recv=131860 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=69632 expected_recv=128751 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=69632 expected_recv=128751 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=69632 expected_recv=131120 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130048 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=69632 expected_recv=130620 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=69632 expected_recv=131087 expected_overflow=0 exact_recv_per_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=69632 expected_recv=131120 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=69632 expected_recv=130620 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=69632 expected_recv=131087 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=69632 expected_recv=130027 expected_overflow=0 exact_recv_per_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=69632 expected_recv=131453 expected_overflow=0 exact_recv_per_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=69632 expected_recv=129750 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=69632 expected_recv=130027 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=69632 expected_recv=131453 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=69632 expected_recv=130993 expected_overflow=0 exact_recv_per_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=69632 expected_recv=129750 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=69632 expected_recv=130993 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=69632 expected_recv=131070 expected_overflow=0 exact_recv_per_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=69632 expected_recv=131129 expected_overflow=0 exact_recv_per_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=69632 expected_recv=131070 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=69632 expected_recv=131129 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=69632 expected_recv=130687 expected_overflow=0 exact_recv_per_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=69632 expected_recv=129178 expected_overflow=0 exact_recv_per_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=69632 expected_recv=130687 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=69632 expected_recv=129178 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=69632 expected_recv=129429 expected_overflow=0 exact_recv_per_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=69632 expected_recv=130971 expected_overflow=0 exact_recv_per_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=69632 expected_recv=129429 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=69632 expected_recv=130971 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[52304, 58428, 53960, 94553, 60013, 78958, 45857, 80215] avg=65536.0000 max=94553 max_rank=3 min=45857 min_rank=6 imbalance_ratio=1.442764 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[52304, 58428, 53960, 94553, 60013, 78958, 45857, 80215] avg=65536.0000 max=94553 max_rank=3 min=45857 min_rank=6 imbalance_ratio=1.442764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[52304, 58428, 53960, 94553, 60013, 78958, 45857, 80215] avg=65536.0000 max=94553 max_rank=3 min=45857 min_rank=6 imbalance_ratio=1.442764 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=69632 expected_recv=129208 expected_overflow=0 exact_recv_per_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62122 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=28360 max_load_reduction_pct=0.299938 moved_to_replicas=109863 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[52304, 58428, 53960, 94553, 60013, 78958, 45857, 80215] avg=65536.0000 max=94553 max_rank=3 min=45857 min_rank=6 imbalance_ratio=1.442764 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=69632 expected_recv=129208 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[52304, 58428, 53960, 94553, 60013, 78958, 45857, 80215] avg=65536.0000 max=94553 max_rank=3 min=45857 min_rank=6 imbalance_ratio=1.442764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62122 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=28360 max_load_reduction_pct=0.299938 moved_to_replicas=109863 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62122 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=28360 max_load_reduction_pct=0.299938 moved_to_replicas=109863 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[52304, 58428, 53960, 94553, 60013, 78958, 45857, 80215] avg=65536.0000 max=94553 max_rank=3 min=45857 min_rank=6 imbalance_ratio=1.442764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[52304, 58428, 53960, 94553, 60013, 78958, 45857, 80215] avg=65536.0000 max=94553 max_rank=3 min=45857 min_rank=6 imbalance_ratio=1.442764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62122 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=28360 max_load_reduction_pct=0.299938 moved_to_replicas=109863 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[65387, 58864, 28669, 98204, 32800, 45681, 65544, 129139] avg=65536.0000 max=129139 max_rank=7 min=28669 min_rank=2 imbalance_ratio=1.970505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62122 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=28360 max_load_reduction_pct=0.299938 moved_to_replicas=109863 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62122 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=28360 max_load_reduction_pct=0.299938 moved_to_replicas=109863 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62122 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=28360 max_load_reduction_pct=0.299938 moved_to_replicas=109863 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[65387, 58864, 28669, 98204, 32800, 45681, 65544, 129139] avg=65536.0000 max=129139 max_rank=7 min=28669 min_rank=2 imbalance_ratio=1.970505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[52304, 58428, 53960, 94553, 60013, 78958, 45857, 80215] avg=65536.0000 max=94553 max_rank=3 min=45857 min_rank=6 imbalance_ratio=1.442764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[65387, 58864, 28669, 98204, 32800, 45681, 65544, 129139] avg=65536.0000 max=129139 max_rank=7 min=28669 min_rank=2 imbalance_ratio=1.970505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[65387, 58864, 28669, 98204, 32800, 45681, 65544, 129139] avg=65536.0000 max=129139 max_rank=7 min=28669 min_rank=2 imbalance_ratio=1.970505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] avg=65536.0000 max=66193 max_rank=5 min=63001 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62946 max_load_reduction_pct=0.487428 moved_to_replicas=190589 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] avg=65536.0000 max=66193 max_rank=5 min=63001 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62946 max_load_reduction_pct=0.487428 moved_to_replicas=190589 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[65387, 58864, 28669, 98204, 32800, 45681, 65544, 129139] avg=65536.0000 max=129139 max_rank=7 min=28669 min_rank=2 imbalance_ratio=1.970505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] avg=65536.0000 max=66193 max_rank=5 min=63001 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62946 max_load_reduction_pct=0.487428 moved_to_replicas=190589 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[62122, 66192, 66192, 66192, 65014, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62122 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=28360 max_load_reduction_pct=0.299938 moved_to_replicas=109863 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[65387, 58864, 28669, 98204, 32800, 45681, 65544, 129139] avg=65536.0000 max=129139 max_rank=7 min=28669 min_rank=2 imbalance_ratio=1.970505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[65387, 58864, 28669, 98204, 32800, 45681, 65544, 129139] avg=65536.0000 max=129139 max_rank=7 min=28669 min_rank=2 imbalance_ratio=1.970505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] avg=65536.0000 max=66193 max_rank=5 min=63001 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62946 max_load_reduction_pct=0.487428 moved_to_replicas=190589 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32804, 98264, 65410, 82534, 81318, 65608, 65559, 32791] avg=65536.0000 max=98264 max_rank=1 min=32791 min_rank=7 imbalance_ratio=1.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] avg=65536.0000 max=66193 max_rank=5 min=63001 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62946 max_load_reduction_pct=0.487428 moved_to_replicas=190589 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] avg=65536.0000 max=66193 max_rank=5 min=63001 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62946 max_load_reduction_pct=0.487428 moved_to_replicas=190589 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] avg=65536.0000 max=66193 max_rank=5 min=63001 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62946 max_load_reduction_pct=0.487428 moved_to_replicas=190589 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32804, 98264, 65410, 82534, 81318, 65608, 65559, 32791] avg=65536.0000 max=98264 max_rank=1 min=32791 min_rank=7 imbalance_ratio=1.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32804, 98264, 65410, 82534, 81318, 65608, 65559, 32791] avg=65536.0000 max=98264 max_rank=1 min=32791 min_rank=7 imbalance_ratio=1.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[65387, 58864, 28669, 98204, 32800, 45681, 65544, 129139] avg=65536.0000 max=129139 max_rank=7 min=28669 min_rank=2 imbalance_ratio=1.970505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32804, 98264, 65410, 82534, 81318, 65608, 65559, 32791] avg=65536.0000 max=98264 max_rank=1 min=32791 min_rank=7 imbalance_ratio=1.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32072 max_load_reduction_pct=0.326386 moved_to_replicas=127152 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32072 max_load_reduction_pct=0.326386 moved_to_replicas=127152 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32804, 98264, 65410, 82534, 81318, 65608, 65559, 32791] avg=65536.0000 max=98264 max_rank=1 min=32791 min_rank=7 imbalance_ratio=1.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32804, 98264, 65410, 82534, 81318, 65608, 65559, 32791] avg=65536.0000 max=98264 max_rank=1 min=32791 min_rank=7 imbalance_ratio=1.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32804, 98264, 65410, 82534, 81318, 65608, 65559, 32791] avg=65536.0000 max=98264 max_rank=1 min=32791 min_rank=7 imbalance_ratio=1.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32072 max_load_reduction_pct=0.326386 moved_to_replicas=127152 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32072 max_load_reduction_pct=0.326386 moved_to_replicas=127152 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[65387, 66191, 65588, 66192, 63001, 66193, 65544, 66192] avg=65536.0000 max=66193 max_rank=5 min=63001 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62946 max_load_reduction_pct=0.487428 moved_to_replicas=190589 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65475, 130210, 65600, 32725, 65615, 33638, 65523, 65502] avg=65536.0000 max=130210 max_rank=1 min=32725 min_rank=3 imbalance_ratio=1.986847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32072 max_load_reduction_pct=0.326386 moved_to_replicas=127152 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32072 max_load_reduction_pct=0.326386 moved_to_replicas=127152 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32072 max_load_reduction_pct=0.326386 moved_to_replicas=127152 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65475, 130210, 65600, 32725, 65615, 33638, 65523, 65502] avg=65536.0000 max=130210 max_rank=1 min=32725 min_rank=3 imbalance_ratio=1.986847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65475, 130210, 65600, 32725, 65615, 33638, 65523, 65502] avg=65536.0000 max=130210 max_rank=1 min=32725 min_rank=3 imbalance_ratio=1.986847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65475, 130210, 65600, 32725, 65615, 33638, 65523, 65502] avg=65536.0000 max=130210 max_rank=1 min=32725 min_rank=3 imbalance_ratio=1.986847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64018 max_load_reduction_pct=0.491652 moved_to_replicas=127662 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32804, 98264, 65410, 82534, 81318, 65608, 65559, 32791] avg=65536.0000 max=98264 max_rank=1 min=32791 min_rank=7 imbalance_ratio=1.499390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64018 max_load_reduction_pct=0.491652 moved_to_replicas=127662 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65475, 130210, 65600, 32725, 65615, 33638, 65523, 65502] avg=65536.0000 max=130210 max_rank=1 min=32725 min_rank=3 imbalance_ratio=1.986847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65475, 130210, 65600, 32725, 65615, 33638, 65523, 65502] avg=65536.0000 max=130210 max_rank=1 min=32725 min_rank=3 imbalance_ratio=1.986847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64018 max_load_reduction_pct=0.491652 moved_to_replicas=127662 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64018 max_load_reduction_pct=0.491652 moved_to_replicas=127662 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65475, 130210, 65600, 32725, 65615, 33638, 65523, 65502] avg=65536.0000 max=130210 max_rank=1 min=32725 min_rank=3 imbalance_ratio=1.986847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[64272, 66192, 65410, 66192, 66192, 65608, 65559, 64863] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32072 max_load_reduction_pct=0.326386 moved_to_replicas=127152 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32800, 88912, 36, 65523, 89471, 98285, 61159, 88102] avg=65536.0000 max=98285 max_rank=5 min=36 min_rank=2 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64018 max_load_reduction_pct=0.491652 moved_to_replicas=127662 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64018 max_load_reduction_pct=0.491652 moved_to_replicas=127662 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32800, 88912, 36, 65523, 89471, 98285, 61159, 88102] avg=65536.0000 max=98285 max_rank=5 min=36 min_rank=2 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64018 max_load_reduction_pct=0.491652 moved_to_replicas=127662 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32800, 88912, 36, 65523, 89471, 98285, 61159, 88102] avg=65536.0000 max=98285 max_rank=5 min=36 min_rank=2 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] avg=65536.0000 max=66196 max_rank=2 min=61613 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32089 max_load_reduction_pct=0.326489 moved_to_replicas=199795 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32800, 88912, 36, 65523, 89471, 98285, 61159, 88102] avg=65536.0000 max=98285 max_rank=5 min=36 min_rank=2 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65475, 130210, 65600, 32725, 65615, 33638, 65523, 65502] avg=65536.0000 max=130210 max_rank=1 min=32725 min_rank=3 imbalance_ratio=1.986847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] avg=65536.0000 max=66196 max_rank=2 min=61613 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32089 max_load_reduction_pct=0.326489 moved_to_replicas=199795 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] avg=65536.0000 max=66196 max_rank=2 min=61613 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32089 max_load_reduction_pct=0.326489 moved_to_replicas=199795 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32800, 88912, 36, 65523, 89471, 98285, 61159, 88102] avg=65536.0000 max=98285 max_rank=5 min=36 min_rank=2 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32800, 88912, 36, 65523, 89471, 98285, 61159, 88102] avg=65536.0000 max=98285 max_rank=5 min=36 min_rank=2 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] avg=65536.0000 max=66196 max_rank=2 min=61613 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32089 max_load_reduction_pct=0.326489 moved_to_replicas=199795 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32800, 88912, 36, 65523, 89471, 98285, 61159, 88102] avg=65536.0000 max=98285 max_rank=5 min=36 min_rank=2 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65475, 66192, 65600, 65484, 65615, 64897, 65523, 65502] avg=65536.0000 max=66192 max_rank=1 min=64897 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64018 max_load_reduction_pct=0.491652 moved_to_replicas=127662 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[32768, 62563, 65091, 98268, 130974, 98307, 33778, 2539] avg=65536.0000 max=130974 max_rank=4 min=2539 min_rank=7 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[32768, 62563, 65091, 98268, 130974, 98307, 33778, 2539] avg=65536.0000 max=130974 max_rank=4 min=2539 min_rank=7 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] avg=65536.0000 max=66196 max_rank=2 min=61613 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32089 max_load_reduction_pct=0.326489 moved_to_replicas=199795 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] avg=65536.0000 max=66196 max_rank=2 min=61613 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32089 max_load_reduction_pct=0.326489 moved_to_replicas=199795 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] avg=65536.0000 max=66196 max_rank=2 min=61613 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32089 max_load_reduction_pct=0.326489 moved_to_replicas=199795 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[32768, 62563, 65091, 98268, 130974, 98307, 33778, 2539] avg=65536.0000 max=130974 max_rank=4 min=2539 min_rank=7 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] avg=65536.0000 max=66192 max_rank=3 min=63746 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64782 max_load_reduction_pct=0.494617 moved_to_replicas=258001 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[32768, 62563, 65091, 98268, 130974, 98307, 33778, 2539] avg=65536.0000 max=130974 max_rank=4 min=2539 min_rank=7 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] avg=65536.0000 max=66192 max_rank=3 min=63746 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64782 max_load_reduction_pct=0.494617 moved_to_replicas=258001 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32800, 88912, 36, 65523, 89471, 98285, 61159, 88102] avg=65536.0000 max=98285 max_rank=5 min=36 min_rank=2 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] avg=65536.0000 max=66192 max_rank=3 min=63746 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64782 max_load_reduction_pct=0.494617 moved_to_replicas=258001 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[32768, 62563, 65091, 98268, 130974, 98307, 33778, 2539] avg=65536.0000 max=130974 max_rank=4 min=2539 min_rank=7 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[32768, 62563, 65091, 98268, 130974, 98307, 33778, 2539] avg=65536.0000 max=130974 max_rank=4 min=2539 min_rank=7 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] avg=65536.0000 max=66192 max_rank=3 min=63746 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64782 max_load_reduction_pct=0.494617 moved_to_replicas=258001 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[32768, 62563, 65091, 98268, 130974, 98307, 33778, 2539] avg=65536.0000 max=130974 max_rank=4 min=2539 min_rank=7 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[66191, 66191, 66196, 65523, 66190, 66192, 61613, 66192] avg=65536.0000 max=66196 max_rank=2 min=61613 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=32089 max_load_reduction_pct=0.326489 moved_to_replicas=199795 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66201, 32784, 32705, 65703, 163218, 32877] avg=65536.0000 max=163218 max_rank=6 min=32705 min_rank=4 imbalance_ratio=2.490509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66201, 32784, 32705, 65703, 163218, 32877] avg=65536.0000 max=163218 max_rank=6 min=32705 min_rank=4 imbalance_ratio=2.490509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] avg=65536.0000 max=66192 max_rank=3 min=63746 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64782 max_load_reduction_pct=0.494617 moved_to_replicas=258001 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] avg=65536.0000 max=66192 max_rank=3 min=63746 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64782 max_load_reduction_pct=0.494617 moved_to_replicas=258001 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] avg=65536.0000 max=66192 max_rank=3 min=63746 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64782 max_load_reduction_pct=0.494617 moved_to_replicas=258001 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66201, 32784, 32705, 65703, 163218, 32877] avg=65536.0000 max=163218 max_rank=6 min=32705 min_rank=4 imbalance_ratio=2.490509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] avg=65536.0000 max=66192 max_rank=2 min=64393 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97026 max_load_reduction_pct=0.594456 moved_to_replicas=194407 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] avg=65536.0000 max=66192 max_rank=2 min=64393 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97026 max_load_reduction_pct=0.594456 moved_to_replicas=194407 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66201, 32784, 32705, 65703, 163218, 32877] avg=65536.0000 max=163218 max_rank=6 min=32705 min_rank=4 imbalance_ratio=2.490509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[32768, 62563, 65091, 98268, 130974, 98307, 33778, 2539] avg=65536.0000 max=130974 max_rank=4 min=2539 min_rank=7 imbalance_ratio=1.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] avg=65536.0000 max=66192 max_rank=2 min=64393 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97026 max_load_reduction_pct=0.594456 moved_to_replicas=194407 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66201, 32784, 32705, 65703, 163218, 32877] avg=65536.0000 max=163218 max_rank=6 min=32705 min_rank=4 imbalance_ratio=2.490509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] avg=65536.0000 max=66192 max_rank=2 min=64393 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97026 max_load_reduction_pct=0.594456 moved_to_replicas=194407 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66201, 32784, 32705, 65703, 163218, 32877] avg=65536.0000 max=163218 max_rank=6 min=32705 min_rank=4 imbalance_ratio=2.490509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66201, 32784, 32705, 65703, 163218, 32877] avg=65536.0000 max=163218 max_rank=6 min=32705 min_rank=4 imbalance_ratio=2.490509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[64790, 63746, 65091, 66192, 66192, 66192, 65893, 66192] avg=65536.0000 max=66192 max_rank=3 min=63746 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64782 max_load_reduction_pct=0.494617 moved_to_replicas=258001 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[90932, 33242, 65526, 171093, 32644, 28, 32835, 97988] avg=65536.0000 max=171093 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.610672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[90932, 33242, 65526, 171093, 32644, 28, 32835, 97988] avg=65536.0000 max=171093 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.610672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] avg=65536.0000 max=66192 max_rank=2 min=64393 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97026 max_load_reduction_pct=0.594456 moved_to_replicas=194407 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] avg=65536.0000 max=66192 max_rank=2 min=64393 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97026 max_load_reduction_pct=0.594456 moved_to_replicas=194407 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[90932, 33242, 65526, 171093, 32644, 28, 32835, 97988] avg=65536.0000 max=171093 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.610672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] avg=65536.0000 max=66192 max_rank=2 min=64393 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97026 max_load_reduction_pct=0.594456 moved_to_replicas=194407 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] avg=65536.0000 max=66192 max_rank=7 min=64203 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=104901 max_load_reduction_pct=0.613123 moved_to_replicas=322931 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] avg=65536.0000 max=66192 max_rank=7 min=64203 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=104901 max_load_reduction_pct=0.613123 moved_to_replicas=322931 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[90932, 33242, 65526, 171093, 32644, 28, 32835, 97988] avg=65536.0000 max=171093 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.610672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66201, 32784, 32705, 65703, 163218, 32877] avg=65536.0000 max=163218 max_rank=6 min=32705 min_rank=4 imbalance_ratio=2.490509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] avg=65536.0000 max=66192 max_rank=7 min=64203 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=104901 max_load_reduction_pct=0.613123 moved_to_replicas=322931 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[90932, 33242, 65526, 171093, 32644, 28, 32835, 97988] avg=65536.0000 max=171093 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.610672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[90932, 33242, 65526, 171093, 32644, 28, 32835, 97988] avg=65536.0000 max=171093 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.610672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] avg=65536.0000 max=66192 max_rank=7 min=64203 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=104901 max_load_reduction_pct=0.613123 moved_to_replicas=322931 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[90932, 33242, 65526, 171093, 32644, 28, 32835, 97988] avg=65536.0000 max=171093 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.610672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[32826, 57652, 69, 98465, 105670, 32825, 130998, 65783] avg=65536.0000 max=130998 max_rank=6 min=69 min_rank=2 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[32826, 57652, 69, 98465, 105670, 32825, 130998, 65783] avg=65536.0000 max=130998 max_rank=6 min=69 min_rank=2 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65262, 65538, 66192, 65543, 65465, 65703, 66192, 64393] avg=65536.0000 max=66192 max_rank=2 min=64393 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97026 max_load_reduction_pct=0.594456 moved_to_replicas=194407 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] avg=65536.0000 max=66192 max_rank=7 min=64203 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=104901 max_load_reduction_pct=0.613123 moved_to_replicas=322931 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] avg=65536.0000 max=66192 max_rank=7 min=64203 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=104901 max_load_reduction_pct=0.613123 moved_to_replicas=322931 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[32826, 57652, 69, 98465, 105670, 32825, 130998, 65783] avg=65536.0000 max=130998 max_rank=6 min=69 min_rank=2 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] avg=65536.0000 max=66192 max_rank=7 min=64203 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=104901 max_load_reduction_pct=0.613123 moved_to_replicas=322931 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] avg=65536.0000 max=66193 max_rank=4 min=63277 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64805 max_load_reduction_pct=0.494702 moved_to_replicas=270447 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] avg=65536.0000 max=66193 max_rank=4 min=63277 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64805 max_load_reduction_pct=0.494702 moved_to_replicas=270447 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[32826, 57652, 69, 98465, 105670, 32825, 130998, 65783] avg=65536.0000 max=130998 max_rank=6 min=69 min_rank=2 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] avg=65536.0000 max=66193 max_rank=4 min=63277 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64805 max_load_reduction_pct=0.494702 moved_to_replicas=270447 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[90932, 33242, 65526, 171093, 32644, 28, 32835, 97988] avg=65536.0000 max=171093 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.610672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[32826, 57652, 69, 98465, 105670, 32825, 130998, 65783] avg=65536.0000 max=130998 max_rank=6 min=69 min_rank=2 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[32826, 57652, 69, 98465, 105670, 32825, 130998, 65783] avg=65536.0000 max=130998 max_rank=6 min=69 min_rank=2 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] avg=65536.0000 max=66193 max_rank=4 min=63277 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64805 max_load_reduction_pct=0.494702 moved_to_replicas=270447 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[113340, 65548, 32786, 32789, 66265, 65712, 32882, 114966] avg=65536.0000 max=114966 max_rank=7 min=32786 min_rank=2 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[32826, 57652, 69, 98465, 105670, 32825, 130998, 65783] avg=65536.0000 max=130998 max_rank=6 min=69 min_rank=2 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[113340, 65548, 32786, 32789, 66265, 65712, 32882, 114966] avg=65536.0000 max=114966 max_rank=7 min=32786 min_rank=2 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[66190, 65038, 65526, 66191, 65402, 65546, 64203, 66192] avg=65536.0000 max=66192 max_rank=7 min=64203 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=104901 max_load_reduction_pct=0.613123 moved_to_replicas=322931 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] avg=65536.0000 max=66193 max_rank=4 min=63277 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64805 max_load_reduction_pct=0.494702 moved_to_replicas=270447 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] avg=65536.0000 max=66193 max_rank=4 min=63277 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64805 max_load_reduction_pct=0.494702 moved_to_replicas=270447 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[113340, 65548, 32786, 32789, 66265, 65712, 32882, 114966] avg=65536.0000 max=114966 max_rank=7 min=32786 min_rank=2 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] avg=65536.0000 max=66193 max_rank=7 min=63264 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=48773 max_load_reduction_pct=0.424238 moved_to_replicas=192731 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] avg=65536.0000 max=66193 max_rank=4 min=63277 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64805 max_load_reduction_pct=0.494702 moved_to_replicas=270447 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] avg=65536.0000 max=66193 max_rank=7 min=63264 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=48773 max_load_reduction_pct=0.424238 moved_to_replicas=192731 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[113340, 65548, 32786, 32789, 66265, 65712, 32882, 114966] avg=65536.0000 max=114966 max_rank=7 min=32786 min_rank=2 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] avg=65536.0000 max=66193 max_rank=7 min=63264 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=48773 max_load_reduction_pct=0.424238 moved_to_replicas=192731 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[32826, 57652, 69, 98465, 105670, 32825, 130998, 65783] avg=65536.0000 max=130998 max_rank=6 min=69 min_rank=2 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[113340, 65548, 32786, 32789, 66265, 65712, 32882, 114966] avg=65536.0000 max=114966 max_rank=7 min=32786 min_rank=2 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[113340, 65548, 32786, 32789, 66265, 65712, 32882, 114966] avg=65536.0000 max=114966 max_rank=7 min=32786 min_rank=2 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[98067, 11052, 32842, 98269, 98271, 77156, 76006, 32625] avg=65536.0000 max=98271 max_rank=4 min=11052 min_rank=1 imbalance_ratio=1.499496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] avg=65536.0000 max=66193 max_rank=7 min=63264 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=48773 max_load_reduction_pct=0.424238 moved_to_replicas=192731 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[113340, 65548, 32786, 32789, 66265, 65712, 32882, 114966] avg=65536.0000 max=114966 max_rank=7 min=32786 min_rank=2 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[98067, 11052, 32842, 98269, 98271, 77156, 76006, 32625] avg=65536.0000 max=98271 max_rank=4 min=11052 min_rank=1 imbalance_ratio=1.499496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[66191, 63277, 64875, 66192, 66193, 65585, 66192, 65783] avg=65536.0000 max=66193 max_rank=4 min=63277 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=64805 max_load_reduction_pct=0.494702 moved_to_replicas=270447 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] avg=65536.0000 max=66193 max_rank=7 min=63264 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=48773 max_load_reduction_pct=0.424238 moved_to_replicas=192731 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[98067, 11052, 32842, 98269, 98271, 77156, 76006, 32625] avg=65536.0000 max=98271 max_rank=4 min=11052 min_rank=1 imbalance_ratio=1.499496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] avg=65536.0000 max=66193 max_rank=7 min=63264 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=48773 max_load_reduction_pct=0.424238 moved_to_replicas=192731 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] avg=65536.0000 max=66192 max_rank=0 min=63909 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32079 max_load_reduction_pct=0.326434 moved_to_replicas=237085 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] avg=65536.0000 max=66193 max_rank=7 min=63264 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=48773 max_load_reduction_pct=0.424238 moved_to_replicas=192731 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] avg=65536.0000 max=66192 max_rank=0 min=63909 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32079 max_load_reduction_pct=0.326434 moved_to_replicas=237085 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[98067, 11052, 32842, 98269, 98271, 77156, 76006, 32625] avg=65536.0000 max=98271 max_rank=4 min=11052 min_rank=1 imbalance_ratio=1.499496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] avg=65536.0000 max=66192 max_rank=0 min=63909 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32079 max_load_reduction_pct=0.326434 moved_to_replicas=237085 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[98067, 11052, 32842, 98269, 98271, 77156, 76006, 32625] avg=65536.0000 max=98271 max_rank=4 min=11052 min_rank=1 imbalance_ratio=1.499496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[113340, 65548, 32786, 32789, 66265, 65712, 32882, 114966] avg=65536.0000 max=114966 max_rank=7 min=32786 min_rank=2 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[98067, 11052, 32842, 98269, 98271, 77156, 76006, 32625] avg=65536.0000 max=98271 max_rank=4 min=11052 min_rank=1 imbalance_ratio=1.499496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[65291, 73196, 65553, 130545, 28, 123165, 923, 65587] avg=65536.0000 max=130545 max_rank=3 min=28 min_rank=4 imbalance_ratio=1.991959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] avg=65536.0000 max=66192 max_rank=0 min=63909 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32079 max_load_reduction_pct=0.326434 moved_to_replicas=237085 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[98067, 11052, 32842, 98269, 98271, 77156, 76006, 32625] avg=65536.0000 max=98271 max_rank=4 min=11052 min_rank=1 imbalance_ratio=1.499496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[65291, 73196, 65553, 130545, 28, 123165, 923, 65587] avg=65536.0000 max=130545 max_rank=3 min=28 min_rank=4 imbalance_ratio=1.991959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] avg=65536.0000 max=66192 max_rank=0 min=63909 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32079 max_load_reduction_pct=0.326434 moved_to_replicas=237085 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65548, 65545, 63264, 66192, 65712, 65642, 66193] avg=65536.0000 max=66193 max_rank=7 min=63264 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=48773 max_load_reduction_pct=0.424238 moved_to_replicas=192731 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[65291, 73196, 65553, 130545, 28, 123165, 923, 65587] avg=65536.0000 max=130545 max_rank=3 min=28 min_rank=4 imbalance_ratio=1.991959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] avg=65536.0000 max=66192 max_rank=0 min=63909 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32079 max_load_reduction_pct=0.326434 moved_to_replicas=237085 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] avg=65536.0000 max=66193 max_rank=5 min=64003 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64352 max_load_reduction_pct=0.492949 moved_to_replicas=256976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] avg=65536.0000 max=66192 max_rank=0 min=63909 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32079 max_load_reduction_pct=0.326434 moved_to_replicas=237085 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] avg=65536.0000 max=66193 max_rank=5 min=64003 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64352 max_load_reduction_pct=0.492949 moved_to_replicas=256976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[65291, 73196, 65553, 130545, 28, 123165, 923, 65587] avg=65536.0000 max=130545 max_rank=3 min=28 min_rank=4 imbalance_ratio=1.991959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] avg=65536.0000 max=66193 max_rank=5 min=64003 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64352 max_load_reduction_pct=0.492949 moved_to_replicas=256976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[65291, 73196, 65553, 130545, 28, 123165, 923, 65587] avg=65536.0000 max=130545 max_rank=3 min=28 min_rank=4 imbalance_ratio=1.991959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[98067, 11052, 32842, 98269, 98271, 77156, 76006, 32625] avg=65536.0000 max=98271 max_rank=4 min=11052 min_rank=1 imbalance_ratio=1.499496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[65291, 73196, 65553, 130545, 28, 123165, 923, 65587] avg=65536.0000 max=130545 max_rank=3 min=28 min_rank=4 imbalance_ratio=1.991959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[65075, 22834, 65570, 68221, 65521, 66580, 100095, 70392] avg=65536.0000 max=100095 max_rank=6 min=22834 min_rank=1 imbalance_ratio=1.527328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] avg=65536.0000 max=66193 max_rank=5 min=64003 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64352 max_load_reduction_pct=0.492949 moved_to_replicas=256976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[65291, 73196, 65553, 130545, 28, 123165, 923, 65587] avg=65536.0000 max=130545 max_rank=3 min=28 min_rank=4 imbalance_ratio=1.991959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[65075, 22834, 65570, 68221, 65521, 66580, 100095, 70392] avg=65536.0000 max=100095 max_rank=6 min=22834 min_rank=1 imbalance_ratio=1.527328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] avg=65536.0000 max=66193 max_rank=5 min=64003 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64352 max_load_reduction_pct=0.492949 moved_to_replicas=256976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[65075, 22834, 65570, 68221, 65521, 66580, 100095, 70392] avg=65536.0000 max=100095 max_rank=6 min=22834 min_rank=1 imbalance_ratio=1.527328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] avg=65536.0000 max=66193 max_rank=5 min=64003 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64352 max_load_reduction_pct=0.492949 moved_to_replicas=256976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63909, 64717, 66192, 66192, 66192, 66192, 64702] avg=65536.0000 max=66192 max_rank=0 min=63909 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32079 max_load_reduction_pct=0.326434 moved_to_replicas=237085 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63352 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33901 max_load_reduction_pct=0.338688 moved_to_replicas=79078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] avg=65536.0000 max=66193 max_rank=5 min=64003 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64352 max_load_reduction_pct=0.492949 moved_to_replicas=256976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63352 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33901 max_load_reduction_pct=0.338688 moved_to_replicas=79078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63352 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33901 max_load_reduction_pct=0.338688 moved_to_replicas=79078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[65075, 22834, 65570, 68221, 65521, 66580, 100095, 70392] avg=65536.0000 max=100095 max_rank=6 min=22834 min_rank=1 imbalance_ratio=1.527328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[65075, 22834, 65570, 68221, 65521, 66580, 100095, 70392] avg=65536.0000 max=100095 max_rank=6 min=22834 min_rank=1 imbalance_ratio=1.527328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[65075, 22834, 65570, 68221, 65521, 66580, 100095, 70392] avg=65536.0000 max=100095 max_rank=6 min=22834 min_rank=1 imbalance_ratio=1.527328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[130924, 3031, 131339, 32930, 65137, 63324, 65558, 32045] avg=65536.0000 max=131339 max_rank=2 min=3031 min_rank=1 imbalance_ratio=2.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[65291, 73196, 65553, 130545, 28, 123165, 923, 65587] avg=65536.0000 max=130545 max_rank=3 min=28 min_rank=4 imbalance_ratio=1.991959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63352 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33901 max_load_reduction_pct=0.338688 moved_to_replicas=79078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[130924, 3031, 131339, 32930, 65137, 63324, 65558, 32045] avg=65536.0000 max=131339 max_rank=2 min=3031 min_rank=1 imbalance_ratio=2.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[65075, 22834, 65570, 68221, 65521, 66580, 100095, 70392] avg=65536.0000 max=100095 max_rank=6 min=22834 min_rank=1 imbalance_ratio=1.527328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63352 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33901 max_load_reduction_pct=0.338688 moved_to_replicas=79078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63352 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33901 max_load_reduction_pct=0.338688 moved_to_replicas=79078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[130924, 3031, 131339, 32930, 65137, 63324, 65558, 32045] avg=65536.0000 max=131339 max_rank=2 min=3031 min_rank=1 imbalance_ratio=2.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] avg=65536.0000 max=66192 max_rank=0 min=64434 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65147 max_load_reduction_pct=0.496022 moved_to_replicas=259603 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[65291, 66192, 65553, 66192, 64003, 66193, 65277, 65587] avg=65536.0000 max=66193 max_rank=5 min=64003 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64352 max_load_reduction_pct=0.492949 moved_to_replicas=256976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] avg=65536.0000 max=66192 max_rank=0 min=64434 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65147 max_load_reduction_pct=0.496022 moved_to_replicas=259603 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63352 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33901 max_load_reduction_pct=0.338688 moved_to_replicas=79078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] avg=65536.0000 max=66192 max_rank=0 min=64434 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65147 max_load_reduction_pct=0.496022 moved_to_replicas=259603 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[130924, 3031, 131339, 32930, 65137, 63324, 65558, 32045] avg=65536.0000 max=131339 max_rank=2 min=3031 min_rank=1 imbalance_ratio=2.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[130924, 3031, 131339, 32930, 65137, 63324, 65558, 32045] avg=65536.0000 max=131339 max_rank=2 min=3031 min_rank=1 imbalance_ratio=2.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[130924, 3031, 131339, 32930, 65137, 63324, 65558, 32045] avg=65536.0000 max=131339 max_rank=2 min=3031 min_rank=1 imbalance_ratio=2.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[95803, 32784, 32204, 33137, 164810, 34548, 65534, 65468] avg=65536.0000 max=164810 max_rank=4 min=32204 min_rank=2 imbalance_ratio=2.514801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[65075, 22834, 65570, 68221, 65521, 66580, 100095, 70392] avg=65536.0000 max=100095 max_rank=6 min=22834 min_rank=1 imbalance_ratio=1.527328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] avg=65536.0000 max=66192 max_rank=0 min=64434 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65147 max_load_reduction_pct=0.496022 moved_to_replicas=259603 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[95803, 32784, 32204, 33137, 164810, 34548, 65534, 65468] avg=65536.0000 max=164810 max_rank=4 min=32204 min_rank=2 imbalance_ratio=2.514801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] avg=65536.0000 max=66192 max_rank=0 min=64434 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65147 max_load_reduction_pct=0.496022 moved_to_replicas=259603 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] avg=65536.0000 max=66192 max_rank=0 min=64434 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65147 max_load_reduction_pct=0.496022 moved_to_replicas=259603 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[130924, 3031, 131339, 32930, 65137, 63324, 65558, 32045] avg=65536.0000 max=131339 max_rank=2 min=3031 min_rank=1 imbalance_ratio=2.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[95803, 32784, 32204, 33137, 164810, 34548, 65534, 65468] avg=65536.0000 max=164810 max_rank=4 min=32204 min_rank=2 imbalance_ratio=2.514801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] avg=65536.0000 max=66192 max_rank=0 min=64504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98618 max_load_reduction_pct=0.598374 moved_to_replicas=255303 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[65075, 63352, 65570, 66192, 65521, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63352 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33901 max_load_reduction_pct=0.338688 moved_to_replicas=79078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] avg=65536.0000 max=66192 max_rank=0 min=64504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98618 max_load_reduction_pct=0.598374 moved_to_replicas=255303 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] avg=65536.0000 max=66192 max_rank=0 min=64504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98618 max_load_reduction_pct=0.598374 moved_to_replicas=255303 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] avg=65536.0000 max=66192 max_rank=0 min=64434 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65147 max_load_reduction_pct=0.496022 moved_to_replicas=259603 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[95803, 32784, 32204, 33137, 164810, 34548, 65534, 65468] avg=65536.0000 max=164810 max_rank=4 min=32204 min_rank=2 imbalance_ratio=2.514801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[95803, 32784, 32204, 33137, 164810, 34548, 65534, 65468] avg=65536.0000 max=164810 max_rank=4 min=32204 min_rank=2 imbalance_ratio=2.514801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[95803, 32784, 32204, 33137, 164810, 34548, 65534, 65468] avg=65536.0000 max=164810 max_rank=4 min=32204 min_rank=2 imbalance_ratio=2.514801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[33159, 65577, 98883, 65230, 65494, 168, 97592, 98185] avg=65536.0000 max=98883 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.508835 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[130924, 3031, 131339, 32930, 65137, 63324, 65558, 32045] avg=65536.0000 max=131339 max_rank=2 min=3031 min_rank=1 imbalance_ratio=2.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[33159, 65577, 98883, 65230, 65494, 168, 97592, 98185] avg=65536.0000 max=98883 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.508835 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] avg=65536.0000 max=66192 max_rank=0 min=64504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98618 max_load_reduction_pct=0.598374 moved_to_replicas=255303 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] avg=65536.0000 max=66192 max_rank=0 min=64504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98618 max_load_reduction_pct=0.598374 moved_to_replicas=255303 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] avg=65536.0000 max=66192 max_rank=0 min=64504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98618 max_load_reduction_pct=0.598374 moved_to_replicas=255303 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[33159, 65577, 98883, 65230, 65494, 168, 97592, 98185] avg=65536.0000 max=98883 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.508835 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64559 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32691 max_load_reduction_pct=0.330603 moved_to_replicas=192362 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[95803, 32784, 32204, 33137, 164810, 34548, 65534, 65468] avg=65536.0000 max=164810 max_rank=4 min=32204 min_rank=2 imbalance_ratio=2.514801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64559 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32691 max_load_reduction_pct=0.330603 moved_to_replicas=192362 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64559 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32691 max_load_reduction_pct=0.330603 moved_to_replicas=192362 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[66192, 66192, 66192, 65685, 65137, 64898, 65558, 64434] avg=65536.0000 max=66192 max_rank=0 min=64434 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65147 max_load_reduction_pct=0.496022 moved_to_replicas=259603 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[33159, 65577, 98883, 65230, 65494, 168, 97592, 98185] avg=65536.0000 max=98883 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.508835 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] avg=65536.0000 max=66192 max_rank=0 min=64504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98618 max_load_reduction_pct=0.598374 moved_to_replicas=255303 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[33159, 65577, 98883, 65230, 65494, 168, 97592, 98185] avg=65536.0000 max=98883 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.508835 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[33159, 65577, 98883, 65230, 65494, 168, 97592, 98185] avg=65536.0000 max=98883 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.508835 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64559 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32691 max_load_reduction_pct=0.330603 moved_to_replicas=192362 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64559 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32691 max_load_reduction_pct=0.330603 moved_to_replicas=192362 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64559 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32691 max_load_reduction_pct=0.330603 moved_to_replicas=192362 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[95803, 32784, 32204, 33137, 164810, 34548, 65534, 65468] avg=65536.0000 max=164810 max_rank=4 min=32204 min_rank=2 imbalance_ratio=2.514801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[33159, 65577, 98883, 65230, 65494, 168, 97592, 98185] avg=65536.0000 max=98883 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.508835 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65543, 64964, 65891, 66192, 64504, 65534, 65468] avg=65536.0000 max=66192 max_rank=0 min=64504 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=98618 max_load_reduction_pct=0.598374 moved_to_replicas=255303 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64559 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32691 max_load_reduction_pct=0.330603 moved_to_replicas=192362 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[33159, 65577, 98883, 65230, 65494, 168, 97592, 98185] avg=65536.0000 max=98883 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.508835 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[64559, 65577, 66192, 65230, 65494, 64852, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64559 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32691 max_load_reduction_pct=0.330603 moved_to_replicas=192362 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=2105328 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=104960 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=2088561 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=121216 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=2100993 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=109184 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=2095169 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=114944 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=2092303 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=117248 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=2099931 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=109952 peak_demand=66196 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=2094453 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=116608 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=2100478 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=109440 peak_demand=66194 required_capacity=1.010040 + [2026-08-30 05:15:27] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 532.0 | throughput per GPU (TFLOP/s/GPU): 626.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.273837E+00 | loss scale: 1.0 | grad norm: 5.595 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=69632 expected_recv=130165 expected_overflow=0 exact_recv_per_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=69632 expected_recv=131010 expected_overflow=0 exact_recv_per_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=69632 expected_recv=127582 expected_overflow=0 exact_recv_per_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=69632 expected_recv=130165 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=69632 expected_recv=131010 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=69632 expected_recv=130282 expected_overflow=0 exact_recv_per_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=69632 expected_recv=127582 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=128512 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=69632 expected_recv=130282 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=69632 expected_recv=131034 expected_overflow=0 exact_recv_per_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=69632 expected_recv=126784 expected_overflow=0 exact_recv_per_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=69632 expected_recv=131034 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=69632 expected_recv=126784 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=69632 expected_recv=130537 expected_overflow=0 exact_recv_per_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=69632 expected_recv=130685 expected_overflow=0 exact_recv_per_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=128128 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=69632 expected_recv=130537 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=69632 expected_recv=130685 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=134656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=134656 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=69632 expected_recv=129576 expected_overflow=0 exact_recv_per_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=69632 expected_recv=128618 expected_overflow=0 exact_recv_per_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=69632 expected_recv=129576 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=69632 expected_recv=128618 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=130048 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=69632 expected_recv=129739 expected_overflow=0 exact_recv_per_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=69632 expected_recv=131107 expected_overflow=0 exact_recv_per_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=69632 expected_recv=129739 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=69632 expected_recv=131107 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=69632 expected_recv=130924 expected_overflow=0 exact_recv_per_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=69632 expected_recv=130951 expected_overflow=0 exact_recv_per_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=69632 expected_recv=130924 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=69632 expected_recv=130951 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=69632 expected_recv=127413 expected_overflow=0 exact_recv_per_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=69632 expected_recv=129752 expected_overflow=0 exact_recv_per_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=69632 expected_recv=127413 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=128640 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=69632 expected_recv=129752 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=69632 expected_recv=130627 expected_overflow=0 exact_recv_per_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=69632 expected_recv=131094 expected_overflow=0 exact_recv_per_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=69632 expected_recv=129779 expected_overflow=0 exact_recv_per_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=69632 expected_recv=130627 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=69632 expected_recv=131094 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=69632 expected_recv=129779 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=69632 expected_recv=128257 expected_overflow=0 exact_recv_per_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=69632 expected_recv=131667 expected_overflow=0 exact_recv_per_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=69632 expected_recv=128257 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=69632 expected_recv=131667 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=129280 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=69632 expected_recv=130419 expected_overflow=0 exact_recv_per_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=69632 expected_recv=127926 expected_overflow=0 exact_recv_per_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=69632 expected_recv=130419 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=69632 expected_recv=127926 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=129408 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=69632 expected_recv=130079 expected_overflow=0 exact_recv_per_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=69632 expected_recv=132380 expected_overflow=0 exact_recv_per_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=69632 expected_recv=130620 expected_overflow=0 exact_recv_per_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=69632 expected_recv=130079 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=69632 expected_recv=132380 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=69632 expected_recv=130620 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=69632 expected_recv=127023 expected_overflow=0 exact_recv_per_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=69632 expected_recv=131029 expected_overflow=0 exact_recv_per_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=69632 expected_recv=127023 actual_valid_recv=128256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=69632 expected_recv=131029 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=128256 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=69632 expected_recv=128606 expected_overflow=0 exact_recv_per_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=69632 expected_recv=128606 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=129792 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=69632 expected_recv=131311 expected_overflow=0 exact_recv_per_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=69632 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=69632 expected_recv=131311 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=69632 expected_recv=132387 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=69632 expected_recv=127100 expected_overflow=0 exact_recv_per_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=69632 expected_recv=131712 expected_overflow=0 exact_recv_per_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=69632 expected_recv=131112 expected_overflow=0 exact_recv_per_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=69632 expected_recv=127100 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=69632 expected_recv=130915 expected_overflow=0 exact_recv_per_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=69632 expected_recv=131712 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=69632 expected_recv=131112 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=128512 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=69632 expected_recv=130915 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=69632 expected_recv=132236 expected_overflow=0 exact_recv_per_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=69632 expected_recv=132236 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=69632 expected_recv=130317 expected_overflow=0 exact_recv_per_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=69632 expected_recv=131655 expected_overflow=0 exact_recv_per_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=69632 expected_recv=129178 expected_overflow=0 exact_recv_per_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=69632 expected_recv=130317 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=69632 expected_recv=131655 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=69632 expected_recv=129178 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=69632 expected_recv=129218 expected_overflow=0 exact_recv_per_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=69632 expected_recv=129218 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=69632 expected_recv=129940 expected_overflow=0 exact_recv_per_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=69632 expected_recv=129940 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=69632 expected_recv=130476 expected_overflow=0 exact_recv_per_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=69632 expected_recv=129865 expected_overflow=0 exact_recv_per_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=69632 expected_recv=130476 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=69632 expected_recv=129865 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=69632 expected_recv=131068 expected_overflow=0 exact_recv_per_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=69632 expected_recv=128169 expected_overflow=0 exact_recv_per_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=69632 expected_recv=131068 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=69632 expected_recv=128169 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=129536 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=69632 expected_recv=129836 expected_overflow=0 exact_recv_per_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=69632 expected_recv=129836 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=69632 expected_recv=130196 expected_overflow=0 exact_recv_per_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=69632 expected_recv=126670 expected_overflow=0 exact_recv_per_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=69632 expected_recv=130196 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=69632 expected_recv=126670 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=128128 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=69632 expected_recv=130743 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=69632 expected_recv=132336 expected_overflow=0 exact_recv_per_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=69632 expected_recv=130743 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=69632 expected_recv=132336 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=69632 expected_recv=129227 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=69632 expected_recv=132114 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=69632 expected_recv=128516 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=69632 expected_recv=130824 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=69632 expected_recv=129227 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=69632 expected_recv=132114 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=69632 expected_recv=128516 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=69632 expected_recv=130824 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=129792 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=69632 expected_recv=125019 expected_overflow=0 exact_recv_per_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=69632 expected_recv=125019 actual_valid_recv=126336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=126336 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=69632 expected_recv=130585 expected_overflow=0 exact_recv_per_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=69632 expected_recv=131050 expected_overflow=0 exact_recv_per_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=134144 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=69632 expected_recv=130585 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=69632 expected_recv=131050 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=69632 expected_recv=130656 expected_overflow=0 exact_recv_per_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=69632 expected_recv=130656 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=69632 expected_recv=130668 expected_overflow=0 exact_recv_per_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=69632 expected_recv=130744 expected_overflow=0 exact_recv_per_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=69632 expected_recv=130192 expected_overflow=0 exact_recv_per_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=69632 expected_recv=130668 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=69632 expected_recv=130744 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=69632 expected_recv=131078 expected_overflow=0 exact_recv_per_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=69632 expected_recv=130192 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=69632 expected_recv=131078 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=69632 expected_recv=129164 expected_overflow=0 exact_recv_per_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=69632 expected_recv=131365 expected_overflow=0 exact_recv_per_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=69632 expected_recv=131060 expected_overflow=0 exact_recv_per_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=69632 expected_recv=129164 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=69632 expected_recv=128812 expected_overflow=0 exact_recv_per_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=69632 expected_recv=131365 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=69632 expected_recv=130827 expected_overflow=0 exact_recv_per_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=69632 expected_recv=131060 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=69632 expected_recv=128812 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=69632 expected_recv=130574 expected_overflow=0 exact_recv_per_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=69632 expected_recv=130827 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=69632 expected_recv=130574 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=69632 expected_recv=130368 expected_overflow=0 exact_recv_per_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=69632 expected_recv=130368 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=69632 expected_recv=130664 expected_overflow=0 exact_recv_per_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=69632 expected_recv=130664 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=69632 expected_recv=128654 expected_overflow=0 exact_recv_per_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=69632 expected_recv=132127 expected_overflow=0 exact_recv_per_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=69632 expected_recv=128654 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=69632 expected_recv=132127 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=129536 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[54659, 63698, 57775, 89607, 65682, 72863, 49787, 70217] avg=65536.0000 max=89607 max_rank=3 min=49787 min_rank=6 imbalance_ratio=1.367294 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[54659, 63698, 57775, 89607, 65682, 72863, 49787, 70217] avg=65536.0000 max=89607 max_rank=3 min=49787 min_rank=6 imbalance_ratio=1.367294 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=69632 expected_recv=129701 expected_overflow=0 exact_recv_per_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=69632 expected_recv=129701 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63698 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23414 max_load_reduction_pct=0.261297 moved_to_replicas=66253 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[54659, 63698, 57775, 89607, 65682, 72863, 49787, 70217] avg=65536.0000 max=89607 max_rank=3 min=49787 min_rank=6 imbalance_ratio=1.367294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[54659, 63698, 57775, 89607, 65682, 72863, 49787, 70217] avg=65536.0000 max=89607 max_rank=3 min=49787 min_rank=6 imbalance_ratio=1.367294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[54659, 63698, 57775, 89607, 65682, 72863, 49787, 70217] avg=65536.0000 max=89607 max_rank=3 min=49787 min_rank=6 imbalance_ratio=1.367294 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63698 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23414 max_load_reduction_pct=0.261297 moved_to_replicas=66253 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[54659, 63698, 57775, 89607, 65682, 72863, 49787, 70217] avg=65536.0000 max=89607 max_rank=3 min=49787 min_rank=6 imbalance_ratio=1.367294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[54659, 63698, 57775, 89607, 65682, 72863, 49787, 70217] avg=65536.0000 max=89607 max_rank=3 min=49787 min_rank=6 imbalance_ratio=1.367294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63698 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23414 max_load_reduction_pct=0.261297 moved_to_replicas=66253 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63698 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23414 max_load_reduction_pct=0.261297 moved_to_replicas=66253 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63698 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23414 max_load_reduction_pct=0.261297 moved_to_replicas=66253 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63698 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23414 max_load_reduction_pct=0.261297 moved_to_replicas=66253 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[65519, 75036, 29897, 87698, 32755, 65244, 65390, 102749] avg=65536.0000 max=102749 max_rank=7 min=29897 min_rank=2 imbalance_ratio=1.567825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[65519, 75036, 29897, 87698, 32755, 65244, 65390, 102749] avg=65536.0000 max=102749 max_rank=7 min=29897 min_rank=2 imbalance_ratio=1.567825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63698 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23414 max_load_reduction_pct=0.261297 moved_to_replicas=66253 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[54659, 63698, 57775, 89607, 65682, 72863, 49787, 70217] avg=65536.0000 max=89607 max_rank=3 min=49787 min_rank=6 imbalance_ratio=1.367294 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] avg=65536.0000 max=66193 max_rank=3 min=63367 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=36556 max_load_reduction_pct=0.355780 moved_to_replicas=134434 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[65519, 75036, 29897, 87698, 32755, 65244, 65390, 102749] avg=65536.0000 max=102749 max_rank=7 min=29897 min_rank=2 imbalance_ratio=1.567825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[65519, 75036, 29897, 87698, 32755, 65244, 65390, 102749] avg=65536.0000 max=102749 max_rank=7 min=29897 min_rank=2 imbalance_ratio=1.567825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[65519, 75036, 29897, 87698, 32755, 65244, 65390, 102749] avg=65536.0000 max=102749 max_rank=7 min=29897 min_rank=2 imbalance_ratio=1.567825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] avg=65536.0000 max=66193 max_rank=3 min=63367 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=36556 max_load_reduction_pct=0.355780 moved_to_replicas=134434 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[65519, 75036, 29897, 87698, 32755, 65244, 65390, 102749] avg=65536.0000 max=102749 max_rank=7 min=29897 min_rank=2 imbalance_ratio=1.567825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[65322, 63698, 64817, 66192, 65682, 66192, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63698 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23414 max_load_reduction_pct=0.261297 moved_to_replicas=66253 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[65519, 75036, 29897, 87698, 32755, 65244, 65390, 102749] avg=65536.0000 max=102749 max_rank=7 min=29897 min_rank=2 imbalance_ratio=1.567825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] avg=65536.0000 max=66193 max_rank=3 min=63367 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=36556 max_load_reduction_pct=0.355780 moved_to_replicas=134434 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] avg=65536.0000 max=66193 max_rank=3 min=63367 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=36556 max_load_reduction_pct=0.355780 moved_to_replicas=134434 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] avg=65536.0000 max=66193 max_rank=3 min=63367 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=36556 max_load_reduction_pct=0.355780 moved_to_replicas=134434 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32539, 98280, 62295, 97620, 98439, 36783, 65553, 32779] avg=65536.0000 max=98439 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.502060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] avg=65536.0000 max=66193 max_rank=3 min=63367 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=36556 max_load_reduction_pct=0.355780 moved_to_replicas=134434 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32539, 98280, 62295, 97620, 98439, 36783, 65553, 32779] avg=65536.0000 max=98439 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.502060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] avg=65536.0000 max=66193 max_rank=3 min=63367 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=36556 max_load_reduction_pct=0.355780 moved_to_replicas=134434 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[65519, 75036, 29897, 87698, 32755, 65244, 65390, 102749] avg=65536.0000 max=102749 max_rank=7 min=29897 min_rank=2 imbalance_ratio=1.567825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] avg=65536.0000 max=66192 max_rank=1 min=64314 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32247 max_load_reduction_pct=0.327584 moved_to_replicas=191554 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32539, 98280, 62295, 97620, 98439, 36783, 65553, 32779] avg=65536.0000 max=98439 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.502060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32539, 98280, 62295, 97620, 98439, 36783, 65553, 32779] avg=65536.0000 max=98439 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.502060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32539, 98280, 62295, 97620, 98439, 36783, 65553, 32779] avg=65536.0000 max=98439 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.502060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] avg=65536.0000 max=66192 max_rank=1 min=64314 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32247 max_load_reduction_pct=0.327584 moved_to_replicas=191554 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32539, 98280, 62295, 97620, 98439, 36783, 65553, 32779] avg=65536.0000 max=98439 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.502060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[65519, 66190, 63367, 66193, 66193, 65244, 65390, 66192] avg=65536.0000 max=66193 max_rank=3 min=63367 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=36556 max_load_reduction_pct=0.355780 moved_to_replicas=134434 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32539, 98280, 62295, 97620, 98439, 36783, 65553, 32779] avg=65536.0000 max=98439 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.502060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65476, 130600, 83676, 32721, 62391, 33208, 52787, 63429] avg=65536.0000 max=130600 max_rank=1 min=32721 min_rank=3 imbalance_ratio=1.992798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] avg=65536.0000 max=66192 max_rank=1 min=64314 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32247 max_load_reduction_pct=0.327584 moved_to_replicas=191554 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] avg=65536.0000 max=66192 max_rank=1 min=64314 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32247 max_load_reduction_pct=0.327584 moved_to_replicas=191554 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] avg=65536.0000 max=66192 max_rank=1 min=64314 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32247 max_load_reduction_pct=0.327584 moved_to_replicas=191554 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] avg=65536.0000 max=66192 max_rank=1 min=64314 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32247 max_load_reduction_pct=0.327584 moved_to_replicas=191554 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] avg=65536.0000 max=66192 max_rank=1 min=64314 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32247 max_load_reduction_pct=0.327584 moved_to_replicas=191554 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65476, 130600, 83676, 32721, 62391, 33208, 52787, 63429] avg=65536.0000 max=130600 max_rank=1 min=32721 min_rank=3 imbalance_ratio=1.992798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] avg=65536.0000 max=66192 max_rank=1 min=63707 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64408 max_load_reduction_pct=0.493170 moved_to_replicas=163786 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32539, 98280, 62295, 97620, 98439, 36783, 65553, 32779] avg=65536.0000 max=98439 max_rank=4 min=32539 min_rank=0 imbalance_ratio=1.502060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65476, 130600, 83676, 32721, 62391, 33208, 52787, 63429] avg=65536.0000 max=130600 max_rank=1 min=32721 min_rank=3 imbalance_ratio=1.992798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65476, 130600, 83676, 32721, 62391, 33208, 52787, 63429] avg=65536.0000 max=130600 max_rank=1 min=32721 min_rank=3 imbalance_ratio=1.992798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65476, 130600, 83676, 32721, 62391, 33208, 52787, 63429] avg=65536.0000 max=130600 max_rank=1 min=32721 min_rank=3 imbalance_ratio=1.992798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65476, 130600, 83676, 32721, 62391, 33208, 52787, 63429] avg=65536.0000 max=130600 max_rank=1 min=32721 min_rank=3 imbalance_ratio=1.992798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] avg=65536.0000 max=66192 max_rank=1 min=63707 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64408 max_load_reduction_pct=0.493170 moved_to_replicas=163786 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65476, 130600, 83676, 32721, 62391, 33208, 52787, 63429] avg=65536.0000 max=130600 max_rank=1 min=32721 min_rank=3 imbalance_ratio=1.992798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[64786, 66192, 64314, 66192, 66192, 66192, 65553, 64867] avg=65536.0000 max=66192 max_rank=1 min=64314 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32247 max_load_reduction_pct=0.327584 moved_to_replicas=191554 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[33174, 98139, 1256, 65543, 65778, 98276, 63938, 98184] avg=65536.0000 max=98276 max_rank=5 min=1256 min_rank=2 imbalance_ratio=1.499573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] avg=65536.0000 max=66192 max_rank=1 min=63707 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64408 max_load_reduction_pct=0.493170 moved_to_replicas=163786 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] avg=65536.0000 max=66192 max_rank=1 min=63707 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64408 max_load_reduction_pct=0.493170 moved_to_replicas=163786 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] avg=65536.0000 max=66192 max_rank=1 min=63707 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64408 max_load_reduction_pct=0.493170 moved_to_replicas=163786 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] avg=65536.0000 max=66192 max_rank=1 min=63707 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64408 max_load_reduction_pct=0.493170 moved_to_replicas=163786 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] avg=65536.0000 max=66192 max_rank=1 min=63707 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64408 max_load_reduction_pct=0.493170 moved_to_replicas=163786 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] avg=65536.0000 max=66192 max_rank=1 min=63938 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32084 max_load_reduction_pct=0.326468 moved_to_replicas=191894 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[33174, 98139, 1256, 65543, 65778, 98276, 63938, 98184] avg=65536.0000 max=98276 max_rank=5 min=1256 min_rank=2 imbalance_ratio=1.499573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65476, 130600, 83676, 32721, 62391, 33208, 52787, 63429] avg=65536.0000 max=130600 max_rank=1 min=32721 min_rank=3 imbalance_ratio=1.992798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[33174, 98139, 1256, 65543, 65778, 98276, 63938, 98184] avg=65536.0000 max=98276 max_rank=5 min=1256 min_rank=2 imbalance_ratio=1.499573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[33174, 98139, 1256, 65543, 65778, 98276, 63938, 98184] avg=65536.0000 max=98276 max_rank=5 min=1256 min_rank=2 imbalance_ratio=1.499573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] avg=65536.0000 max=66192 max_rank=1 min=63938 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32084 max_load_reduction_pct=0.326468 moved_to_replicas=191894 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[33174, 98139, 1256, 65543, 65778, 98276, 63938, 98184] avg=65536.0000 max=98276 max_rank=5 min=1256 min_rank=2 imbalance_ratio=1.499573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[33174, 98139, 1256, 65543, 65778, 98276, 63938, 98184] avg=65536.0000 max=98276 max_rank=5 min=1256 min_rank=2 imbalance_ratio=1.499573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[33174, 98139, 1256, 65543, 65778, 98276, 63938, 98184] avg=65536.0000 max=98276 max_rank=5 min=1256 min_rank=2 imbalance_ratio=1.499573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65476, 66192, 66192, 65473, 66192, 64864, 66192, 63707] avg=65536.0000 max=66192 max_rank=1 min=63707 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64408 max_load_reduction_pct=0.493170 moved_to_replicas=163786 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[32798, 32957, 53840, 98278, 131026, 98318, 44301, 32770] avg=65536.0000 max=131026 max_rank=4 min=32770 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] avg=65536.0000 max=66192 max_rank=1 min=63938 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32084 max_load_reduction_pct=0.326468 moved_to_replicas=191894 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] avg=65536.0000 max=66192 max_rank=1 min=63938 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32084 max_load_reduction_pct=0.326468 moved_to_replicas=191894 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] avg=65536.0000 max=66192 max_rank=1 min=63938 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32084 max_load_reduction_pct=0.326468 moved_to_replicas=191894 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] avg=65536.0000 max=66192 max_rank=1 min=63938 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32084 max_load_reduction_pct=0.326468 moved_to_replicas=191894 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] avg=65536.0000 max=66192 max_rank=1 min=63938 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32084 max_load_reduction_pct=0.326468 moved_to_replicas=191894 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] avg=65536.0000 max=66192 max_rank=3 min=64037 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=258086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[32798, 32957, 53840, 98278, 131026, 98318, 44301, 32770] avg=65536.0000 max=131026 max_rank=4 min=32770 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[33174, 98139, 1256, 65543, 65778, 98276, 63938, 98184] avg=65536.0000 max=98276 max_rank=5 min=1256 min_rank=2 imbalance_ratio=1.499573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[32798, 32957, 53840, 98278, 131026, 98318, 44301, 32770] avg=65536.0000 max=131026 max_rank=4 min=32770 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32798, 32957, 53840, 98278, 131026, 98318, 44301, 32770] avg=65536.0000 max=131026 max_rank=4 min=32770 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] avg=65536.0000 max=66192 max_rank=3 min=64037 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=258086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32798, 32957, 53840, 98278, 131026, 98318, 44301, 32770] avg=65536.0000 max=131026 max_rank=4 min=32770 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[32798, 32957, 53840, 98278, 131026, 98318, 44301, 32770] avg=65536.0000 max=131026 max_rank=4 min=32770 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[65166, 66192, 65287, 65543, 65778, 66192, 63938, 66192] avg=65536.0000 max=66192 max_rank=1 min=63938 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32084 max_load_reduction_pct=0.326468 moved_to_replicas=191894 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32947, 97867, 98082, 32793, 25055, 52110, 127135, 58299] avg=65536.0000 max=127135 max_rank=6 min=25055 min_rank=4 imbalance_ratio=1.939926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] avg=65536.0000 max=66192 max_rank=3 min=64037 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=258086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32798, 32957, 53840, 98278, 131026, 98318, 44301, 32770] avg=65536.0000 max=131026 max_rank=4 min=32770 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] avg=65536.0000 max=66192 max_rank=3 min=64037 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=258086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] avg=65536.0000 max=66192 max_rank=3 min=64037 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=258086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] avg=65536.0000 max=66192 max_rank=3 min=64037 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=258086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=62300 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=60942 max_load_reduction_pct=0.479349 moved_to_replicas=247456 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] avg=65536.0000 max=66192 max_rank=3 min=64037 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=258086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32947, 97867, 98082, 32793, 25055, 52110, 127135, 58299] avg=65536.0000 max=127135 max_rank=6 min=25055 min_rank=4 imbalance_ratio=1.939926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[32798, 32957, 53840, 98278, 131026, 98318, 44301, 32770] avg=65536.0000 max=131026 max_rank=4 min=32770 min_rank=7 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32947, 97867, 98082, 32793, 25055, 52110, 127135, 58299] avg=65536.0000 max=127135 max_rank=6 min=25055 min_rank=4 imbalance_ratio=1.939926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32947, 97867, 98082, 32793, 25055, 52110, 127135, 58299] avg=65536.0000 max=127135 max_rank=6 min=25055 min_rank=4 imbalance_ratio=1.939926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32947, 97867, 98082, 32793, 25055, 52110, 127135, 58299] avg=65536.0000 max=127135 max_rank=6 min=25055 min_rank=4 imbalance_ratio=1.939926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32947, 97867, 98082, 32793, 25055, 52110, 127135, 58299] avg=65536.0000 max=127135 max_rank=6 min=25055 min_rank=4 imbalance_ratio=1.939926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=62300 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=60942 max_load_reduction_pct=0.479349 moved_to_replicas=247456 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[65612, 52128, 110433, 165022, 32719, 29, 32782, 65563] avg=65536.0000 max=165022 max_rank=3 min=29 min_rank=5 imbalance_ratio=2.518036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=62300 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=60942 max_load_reduction_pct=0.479349 moved_to_replicas=247456 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[64872, 65083, 64037, 66192, 66192, 66192, 66190, 65530] avg=65536.0000 max=66192 max_rank=3 min=64037 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=258086 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32947, 97867, 98082, 32793, 25055, 52110, 127135, 58299] avg=65536.0000 max=127135 max_rank=6 min=25055 min_rank=4 imbalance_ratio=1.939926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=62300 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=60942 max_load_reduction_pct=0.479349 moved_to_replicas=247456 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=62300 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=60942 max_load_reduction_pct=0.479349 moved_to_replicas=247456 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=62300 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=60942 max_load_reduction_pct=0.479349 moved_to_replicas=247456 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] avg=65536.0000 max=66194 max_rank=2 min=63608 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=98828 max_load_reduction_pct=0.598878 moved_to_replicas=285617 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=62300 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=60942 max_load_reduction_pct=0.479349 moved_to_replicas=247456 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[65612, 52128, 110433, 165022, 32719, 29, 32782, 65563] avg=65536.0000 max=165022 max_rank=3 min=29 min_rank=5 imbalance_ratio=2.518036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[65612, 52128, 110433, 165022, 32719, 29, 32782, 65563] avg=65536.0000 max=165022 max_rank=3 min=29 min_rank=5 imbalance_ratio=2.518036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32947, 97867, 98082, 32793, 25055, 52110, 127135, 58299] avg=65536.0000 max=127135 max_rank=6 min=25055 min_rank=4 imbalance_ratio=1.939926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[65612, 52128, 110433, 165022, 32719, 29, 32782, 65563] avg=65536.0000 max=165022 max_rank=3 min=29 min_rank=5 imbalance_ratio=2.518036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[65612, 52128, 110433, 165022, 32719, 29, 32782, 65563] avg=65536.0000 max=165022 max_rank=3 min=29 min_rank=5 imbalance_ratio=2.518036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[65612, 52128, 110433, 165022, 32719, 29, 32782, 65563] avg=65536.0000 max=165022 max_rank=3 min=29 min_rank=5 imbalance_ratio=2.518036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] avg=65536.0000 max=66194 max_rank=2 min=63608 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=98828 max_load_reduction_pct=0.598878 moved_to_replicas=285617 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[32779, 64508, 102, 66492, 71133, 60283, 131291, 97700] avg=65536.0000 max=131291 max_rank=6 min=102 min_rank=2 imbalance_ratio=2.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] avg=65536.0000 max=66194 max_rank=2 min=63608 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=98828 max_load_reduction_pct=0.598878 moved_to_replicas=285617 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[64837, 66192, 66192, 62300, 66192, 66191, 66191, 66193] avg=65536.0000 max=66193 max_rank=7 min=62300 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=60942 max_load_reduction_pct=0.479349 moved_to_replicas=247456 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[65612, 52128, 110433, 165022, 32719, 29, 32782, 65563] avg=65536.0000 max=165022 max_rank=3 min=29 min_rank=5 imbalance_ratio=2.518036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] avg=65536.0000 max=66194 max_rank=2 min=63608 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=98828 max_load_reduction_pct=0.598878 moved_to_replicas=285617 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] avg=65536.0000 max=66194 max_rank=2 min=63608 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=98828 max_load_reduction_pct=0.598878 moved_to_replicas=285617 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] avg=65536.0000 max=66194 max_rank=2 min=63608 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=98828 max_load_reduction_pct=0.598878 moved_to_replicas=285617 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64508 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65099 max_load_reduction_pct=0.495837 moved_to_replicas=203379 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] avg=65536.0000 max=66194 max_rank=2 min=63608 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=98828 max_load_reduction_pct=0.598878 moved_to_replicas=285617 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[32779, 64508, 102, 66492, 71133, 60283, 131291, 97700] avg=65536.0000 max=131291 max_rank=6 min=102 min_rank=2 imbalance_ratio=2.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[32779, 64508, 102, 66492, 71133, 60283, 131291, 97700] avg=65536.0000 max=131291 max_rank=6 min=102 min_rank=2 imbalance_ratio=2.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[32779, 64508, 102, 66492, 71133, 60283, 131291, 97700] avg=65536.0000 max=131291 max_rank=6 min=102 min_rank=2 imbalance_ratio=2.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[65612, 52128, 110433, 165022, 32719, 29, 32782, 65563] avg=65536.0000 max=165022 max_rank=3 min=29 min_rank=5 imbalance_ratio=2.518036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[32779, 64508, 102, 66492, 71133, 60283, 131291, 97700] avg=65536.0000 max=131291 max_rank=6 min=102 min_rank=2 imbalance_ratio=2.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[32779, 64508, 102, 66492, 71133, 60283, 131291, 97700] avg=65536.0000 max=131291 max_rank=6 min=102 min_rank=2 imbalance_ratio=2.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64508 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65099 max_load_reduction_pct=0.495837 moved_to_replicas=203379 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65533, 32829, 32764, 75974, 87406, 33096, 98383] avg=65536.0000 max=98383 max_rank=7 min=32764 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64508 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65099 max_load_reduction_pct=0.495837 moved_to_replicas=203379 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[32779, 64508, 102, 66492, 71133, 60283, 131291, 97700] avg=65536.0000 max=131291 max_rank=6 min=102 min_rank=2 imbalance_ratio=2.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64508 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65099 max_load_reduction_pct=0.495837 moved_to_replicas=203379 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[65612, 63608, 66194, 66192, 65470, 66129, 65520, 65563] avg=65536.0000 max=66194 max_rank=2 min=63608 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=98828 max_load_reduction_pct=0.598878 moved_to_replicas=285617 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64508 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65099 max_load_reduction_pct=0.495837 moved_to_replicas=203379 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64508 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65099 max_load_reduction_pct=0.495837 moved_to_replicas=203379 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] avg=65536.0000 max=66192 max_rank=0 min=64094 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190709 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64508 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65099 max_load_reduction_pct=0.495837 moved_to_replicas=203379 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65533, 32829, 32764, 75974, 87406, 33096, 98383] avg=65536.0000 max=98383 max_rank=7 min=32764 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65533, 32829, 32764, 75974, 87406, 33096, 98383] avg=65536.0000 max=98383 max_rank=7 min=32764 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65533, 32829, 32764, 75974, 87406, 33096, 98383] avg=65536.0000 max=98383 max_rank=7 min=32764 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65533, 32829, 32764, 75974, 87406, 33096, 98383] avg=65536.0000 max=98383 max_rank=7 min=32764 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[32779, 64508, 102, 66492, 71133, 60283, 131291, 97700] avg=65536.0000 max=131291 max_rank=6 min=102 min_rank=2 imbalance_ratio=2.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[98217, 780, 32888, 98628, 98239, 66144, 94533, 34859] avg=65536.0000 max=98628 max_rank=3 min=780 min_rank=1 imbalance_ratio=1.504944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65533, 32829, 32764, 75974, 87406, 33096, 98383] avg=65536.0000 max=98383 max_rank=7 min=32764 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] avg=65536.0000 max=66192 max_rank=0 min=64094 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190709 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] avg=65536.0000 max=66192 max_rank=0 min=64094 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190709 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] avg=65536.0000 max=66192 max_rank=0 min=64094 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190709 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65533, 32829, 32764, 75974, 87406, 33096, 98383] avg=65536.0000 max=98383 max_rank=7 min=32764 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] avg=65536.0000 max=66192 max_rank=0 min=64094 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190709 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65121, 64508, 64667, 66192, 66191, 65225, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64508 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65099 max_load_reduction_pct=0.495837 moved_to_replicas=203379 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] avg=65536.0000 max=66193 max_rank=6 min=63199 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32435 max_load_reduction_pct=0.328862 moved_to_replicas=249973 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] avg=65536.0000 max=66192 max_rank=0 min=64094 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190709 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] avg=65536.0000 max=66192 max_rank=0 min=64094 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190709 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[98217, 780, 32888, 98628, 98239, 66144, 94533, 34859] avg=65536.0000 max=98628 max_rank=3 min=780 min_rank=1 imbalance_ratio=1.504944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[98217, 780, 32888, 98628, 98239, 66144, 94533, 34859] avg=65536.0000 max=98628 max_rank=3 min=780 min_rank=1 imbalance_ratio=1.504944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[98217, 780, 32888, 98628, 98239, 66144, 94533, 34859] avg=65536.0000 max=98628 max_rank=3 min=780 min_rank=1 imbalance_ratio=1.504944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[98217, 780, 32888, 98628, 98239, 66144, 94533, 34859] avg=65536.0000 max=98628 max_rank=3 min=780 min_rank=1 imbalance_ratio=1.504944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65533, 32829, 32764, 75974, 87406, 33096, 98383] avg=65536.0000 max=98383 max_rank=7 min=32764 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[65345, 34442, 98122, 130219, 27, 130400, 312, 65421] avg=65536.0000 max=130400 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.989746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[98217, 780, 32888, 98628, 98239, 66144, 94533, 34859] avg=65536.0000 max=98628 max_rank=3 min=780 min_rank=1 imbalance_ratio=1.504944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] avg=65536.0000 max=66193 max_rank=6 min=63199 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32435 max_load_reduction_pct=0.328862 moved_to_replicas=249973 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] avg=65536.0000 max=66193 max_rank=6 min=63199 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32435 max_load_reduction_pct=0.328862 moved_to_replicas=249973 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[98217, 780, 32888, 98628, 98239, 66144, 94533, 34859] avg=65536.0000 max=98628 max_rank=3 min=780 min_rank=1 imbalance_ratio=1.504944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] avg=65536.0000 max=66193 max_rank=6 min=63199 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32435 max_load_reduction_pct=0.328862 moved_to_replicas=249973 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] avg=65536.0000 max=66193 max_rank=6 min=63199 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32435 max_load_reduction_pct=0.328862 moved_to_replicas=249973 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65533, 64940, 64955, 66191, 66191, 64094, 66192] avg=65536.0000 max=66192 max_rank=0 min=64094 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190709 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] avg=65536.0000 max=66192 max_rank=1 min=64235 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64208 max_load_reduction_pct=0.492393 moved_to_replicas=320517 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] avg=65536.0000 max=66193 max_rank=6 min=63199 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32435 max_load_reduction_pct=0.328862 moved_to_replicas=249973 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] avg=65536.0000 max=66193 max_rank=6 min=63199 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32435 max_load_reduction_pct=0.328862 moved_to_replicas=249973 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[65345, 34442, 98122, 130219, 27, 130400, 312, 65421] avg=65536.0000 max=130400 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.989746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[65345, 34442, 98122, 130219, 27, 130400, 312, 65421] avg=65536.0000 max=130400 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.989746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[65345, 34442, 98122, 130219, 27, 130400, 312, 65421] avg=65536.0000 max=130400 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.989746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[65345, 34442, 98122, 130219, 27, 130400, 312, 65421] avg=65536.0000 max=130400 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.989746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[98217, 780, 32888, 98628, 98239, 66144, 94533, 34859] avg=65536.0000 max=98628 max_rank=3 min=780 min_rank=1 imbalance_ratio=1.504944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[32483, 3952, 37539, 97903, 65330, 123133, 98416, 65532] avg=65536.0000 max=123133 max_rank=5 min=3952 min_rank=1 imbalance_ratio=1.878860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[65345, 34442, 98122, 130219, 27, 130400, 312, 65421] avg=65536.0000 max=130400 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.989746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] avg=65536.0000 max=66192 max_rank=1 min=64235 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64208 max_load_reduction_pct=0.492393 moved_to_replicas=320517 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[65345, 34442, 98122, 130219, 27, 130400, 312, 65421] avg=65536.0000 max=130400 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.989746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] avg=65536.0000 max=66192 max_rank=1 min=64235 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64208 max_load_reduction_pct=0.492393 moved_to_replicas=320517 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] avg=65536.0000 max=66192 max_rank=1 min=64235 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64208 max_load_reduction_pct=0.492393 moved_to_replicas=320517 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] avg=65536.0000 max=66192 max_rank=1 min=64235 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64208 max_load_reduction_pct=0.492393 moved_to_replicas=320517 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65241, 64935, 66192, 66192, 66144, 66193, 63199] avg=65536.0000 max=66193 max_rank=6 min=63199 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32435 max_load_reduction_pct=0.328862 moved_to_replicas=249973 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] avg=65536.0000 max=66192 max_rank=1 min=62466 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=56941 max_load_reduction_pct=0.462435 moved_to_replicas=240070 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] avg=65536.0000 max=66192 max_rank=1 min=64235 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64208 max_load_reduction_pct=0.492393 moved_to_replicas=320517 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] avg=65536.0000 max=66192 max_rank=1 min=64235 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64208 max_load_reduction_pct=0.492393 moved_to_replicas=320517 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[32483, 3952, 37539, 97903, 65330, 123133, 98416, 65532] avg=65536.0000 max=123133 max_rank=5 min=3952 min_rank=1 imbalance_ratio=1.878860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[32483, 3952, 37539, 97903, 65330, 123133, 98416, 65532] avg=65536.0000 max=123133 max_rank=5 min=3952 min_rank=1 imbalance_ratio=1.878860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[32483, 3952, 37539, 97903, 65330, 123133, 98416, 65532] avg=65536.0000 max=123133 max_rank=5 min=3952 min_rank=1 imbalance_ratio=1.878860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[32483, 3952, 37539, 97903, 65330, 123133, 98416, 65532] avg=65536.0000 max=123133 max_rank=5 min=3952 min_rank=1 imbalance_ratio=1.878860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[65345, 34442, 98122, 130219, 27, 130400, 312, 65421] avg=65536.0000 max=130400 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.989746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[102192, 32808, 131265, 32860, 33384, 64297, 96052, 31430] avg=65536.0000 max=131265 max_rank=2 min=31430 min_rank=7 imbalance_ratio=2.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] avg=65536.0000 max=66192 max_rank=1 min=62466 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=56941 max_load_reduction_pct=0.462435 moved_to_replicas=240070 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[32483, 3952, 37539, 97903, 65330, 123133, 98416, 65532] avg=65536.0000 max=123133 max_rank=5 min=3952 min_rank=1 imbalance_ratio=1.878860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[32483, 3952, 37539, 97903, 65330, 123133, 98416, 65532] avg=65536.0000 max=123133 max_rank=5 min=3952 min_rank=1 imbalance_ratio=1.878860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] avg=65536.0000 max=66192 max_rank=1 min=62466 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=56941 max_load_reduction_pct=0.462435 moved_to_replicas=240070 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] avg=65536.0000 max=66192 max_rank=1 min=62466 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=56941 max_load_reduction_pct=0.462435 moved_to_replicas=240070 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] avg=65536.0000 max=66192 max_rank=1 min=62466 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=56941 max_load_reduction_pct=0.462435 moved_to_replicas=240070 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[65345, 66192, 66192, 66192, 64235, 66192, 64519, 65421] avg=65536.0000 max=66192 max_rank=1 min=64235 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64208 max_load_reduction_pct=0.492393 moved_to_replicas=320517 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] avg=65536.0000 max=66192 max_rank=0 min=64297 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.495738 moved_to_replicas=259593 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] avg=65536.0000 max=66192 max_rank=1 min=62466 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=56941 max_load_reduction_pct=0.462435 moved_to_replicas=240070 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] avg=65536.0000 max=66192 max_rank=1 min=62466 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=56941 max_load_reduction_pct=0.462435 moved_to_replicas=240070 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[102192, 32808, 131265, 32860, 33384, 64297, 96052, 31430] avg=65536.0000 max=131265 max_rank=2 min=31430 min_rank=7 imbalance_ratio=2.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[102192, 32808, 131265, 32860, 33384, 64297, 96052, 31430] avg=65536.0000 max=131265 max_rank=2 min=31430 min_rank=7 imbalance_ratio=2.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[102192, 32808, 131265, 32860, 33384, 64297, 96052, 31430] avg=65536.0000 max=131265 max_rank=2 min=31430 min_rank=7 imbalance_ratio=2.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[102192, 32808, 131265, 32860, 33384, 64297, 96052, 31430] avg=65536.0000 max=131265 max_rank=2 min=31430 min_rank=7 imbalance_ratio=2.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[32483, 3952, 37539, 97903, 65330, 123133, 98416, 65532] avg=65536.0000 max=123133 max_rank=5 min=3952 min_rank=1 imbalance_ratio=1.878860 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[95205, 32790, 4027, 32779, 163347, 65217, 65529, 65394] avg=65536.0000 max=163347 max_rank=4 min=4027 min_rank=2 imbalance_ratio=2.492477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] avg=65536.0000 max=66192 max_rank=0 min=64297 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.495738 moved_to_replicas=259593 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[102192, 32808, 131265, 32860, 33384, 64297, 96052, 31430] avg=65536.0000 max=131265 max_rank=2 min=31430 min_rank=7 imbalance_ratio=2.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] avg=65536.0000 max=66192 max_rank=0 min=64297 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.495738 moved_to_replicas=259593 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] avg=65536.0000 max=66192 max_rank=0 min=64297 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.495738 moved_to_replicas=259593 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[102192, 32808, 131265, 32860, 33384, 64297, 96052, 31430] avg=65536.0000 max=131265 max_rank=2 min=31430 min_rank=7 imbalance_ratio=2.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] avg=65536.0000 max=66192 max_rank=0 min=64297 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.495738 moved_to_replicas=259593 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[62466, 66192, 66192, 66192, 65330, 66192, 66192, 65532] avg=65536.0000 max=66192 max_rank=1 min=62466 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=56941 max_load_reduction_pct=0.462435 moved_to_replicas=240070 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] avg=65536.0000 max=66192 max_rank=0 min=64431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97155 max_load_reduction_pct=0.594777 moved_to_replicas=251801 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] avg=65536.0000 max=66192 max_rank=0 min=64297 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.495738 moved_to_replicas=259593 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[95205, 32790, 4027, 32779, 163347, 65217, 65529, 65394] avg=65536.0000 max=163347 max_rank=4 min=4027 min_rank=2 imbalance_ratio=2.492477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] avg=65536.0000 max=66192 max_rank=0 min=64297 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.495738 moved_to_replicas=259593 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[95205, 32790, 4027, 32779, 163347, 65217, 65529, 65394] avg=65536.0000 max=163347 max_rank=4 min=4027 min_rank=2 imbalance_ratio=2.492477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[95205, 32790, 4027, 32779, 163347, 65217, 65529, 65394] avg=65536.0000 max=163347 max_rank=4 min=4027 min_rank=2 imbalance_ratio=2.492477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[95205, 32790, 4027, 32779, 163347, 65217, 65529, 65394] avg=65536.0000 max=163347 max_rank=4 min=4027 min_rank=2 imbalance_ratio=2.492477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[102192, 32808, 131265, 32860, 33384, 64297, 96052, 31430] avg=65536.0000 max=131265 max_rank=2 min=31430 min_rank=7 imbalance_ratio=2.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[33725, 66022, 98597, 65211, 64373, 627, 97529, 98204] avg=65536.0000 max=98597 max_rank=2 min=627 min_rank=5 imbalance_ratio=1.504471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] avg=65536.0000 max=66192 max_rank=0 min=64431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97155 max_load_reduction_pct=0.594777 moved_to_replicas=251801 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[95205, 32790, 4027, 32779, 163347, 65217, 65529, 65394] avg=65536.0000 max=163347 max_rank=4 min=4027 min_rank=2 imbalance_ratio=2.492477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] avg=65536.0000 max=66192 max_rank=0 min=64431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97155 max_load_reduction_pct=0.594777 moved_to_replicas=251801 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] avg=65536.0000 max=66192 max_rank=0 min=64431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97155 max_load_reduction_pct=0.594777 moved_to_replicas=251801 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[95205, 32790, 4027, 32779, 163347, 65217, 65529, 65394] avg=65536.0000 max=163347 max_rank=4 min=4027 min_rank=2 imbalance_ratio=2.492477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] avg=65536.0000 max=66192 max_rank=0 min=64431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97155 max_load_reduction_pct=0.594777 moved_to_replicas=251801 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.328661 moved_to_replicas=191637 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[66192, 65123, 66192, 65615, 66192, 64297, 66192, 64485] avg=65536.0000 max=66192 max_rank=0 min=64297 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65073 max_load_reduction_pct=0.495738 moved_to_replicas=259593 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] avg=65536.0000 max=66192 max_rank=0 min=64431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97155 max_load_reduction_pct=0.594777 moved_to_replicas=251801 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[33725, 66022, 98597, 65211, 64373, 627, 97529, 98204] avg=65536.0000 max=98597 max_rank=2 min=627 min_rank=5 imbalance_ratio=1.504471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] avg=65536.0000 max=66192 max_rank=0 min=64431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97155 max_load_reduction_pct=0.594777 moved_to_replicas=251801 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[33725, 66022, 98597, 65211, 64373, 627, 97529, 98204] avg=65536.0000 max=98597 max_rank=2 min=627 min_rank=5 imbalance_ratio=1.504471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[33725, 66022, 98597, 65211, 64373, 627, 97529, 98204] avg=65536.0000 max=98597 max_rank=2 min=627 min_rank=5 imbalance_ratio=1.504471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[33725, 66022, 98597, 65211, 64373, 627, 97529, 98204] avg=65536.0000 max=98597 max_rank=2 min=627 min_rank=5 imbalance_ratio=1.504471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[95205, 32790, 4027, 32779, 163347, 65217, 65529, 65394] avg=65536.0000 max=163347 max_rank=4 min=4027 min_rank=2 imbalance_ratio=2.492477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.328661 moved_to_replicas=191637 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.328661 moved_to_replicas=191637 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[33725, 66022, 98597, 65211, 64373, 627, 97529, 98204] avg=65536.0000 max=98597 max_rank=2 min=627 min_rank=5 imbalance_ratio=1.504471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[33725, 66022, 98597, 65211, 64373, 627, 97529, 98204] avg=65536.0000 max=98597 max_rank=2 min=627 min_rank=5 imbalance_ratio=1.504471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.328661 moved_to_replicas=191637 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.328661 moved_to_replicas=191637 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[66192, 64431, 65799, 65534, 66192, 65217, 65529, 65394] avg=65536.0000 max=66192 max_rank=0 min=64431 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97155 max_load_reduction_pct=0.594777 moved_to_replicas=251801 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.328661 moved_to_replicas=191637 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.328661 moved_to_replicas=191637 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[33725, 66022, 98597, 65211, 64373, 627, 97529, 98204] avg=65536.0000 max=98597 max_rank=2 min=627 min_rank=5 imbalance_ratio=1.504471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[65062, 66022, 66192, 65211, 64373, 65044, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64373 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.328661 moved_to_replicas=191637 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=2102624 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=106368 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=2101670 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=107648 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=2093266 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=116224 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=2102817 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=105600 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=2091070 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=117888 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=2091357 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=117888 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=2091055 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=117120 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=2103357 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=105472 peak_demand=66193 required_capacity=1.010025 + [2026-08-30 05:15:27] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 522.8 | throughput per GPU (TFLOP/s/GPU): 637.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.112632E+00 | loss scale: 1.0 | grad norm: 5.427 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=69632 expected_recv=130538 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=69632 expected_recv=128987 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=69632 expected_recv=130518 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=69632 expected_recv=129288 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=69632 expected_recv=130538 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=69632 expected_recv=128987 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=69632 expected_recv=132093 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=69632 expected_recv=130518 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=69632 expected_recv=132093 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=69632 expected_recv=129288 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=69632 expected_recv=127213 expected_overflow=0 exact_recv_per_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=69632 expected_recv=129405 expected_overflow=0 exact_recv_per_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=69632 expected_recv=127213 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=69632 expected_recv=130039 expected_overflow=0 exact_recv_per_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=69632 expected_recv=129405 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=128128 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=69632 expected_recv=130039 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=69632 expected_recv=128449 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=69632 expected_recv=128449 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=69632 expected_recv=131086 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=130048 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=69632 expected_recv=131086 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=69632 expected_recv=128434 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=69632 expected_recv=131073 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=69632 expected_recv=124143 expected_overflow=0 exact_recv_per_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=69632 expected_recv=128434 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=69632 expected_recv=131073 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=69632 expected_recv=131744 expected_overflow=0 exact_recv_per_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=129664 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=69632 expected_recv=124143 actual_valid_recv=125184 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=69632 expected_recv=131744 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=125184 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=69632 expected_recv=130772 expected_overflow=0 exact_recv_per_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=69632 expected_recv=130772 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=69632 expected_recv=127113 expected_overflow=0 exact_recv_per_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=69632 expected_recv=130536 expected_overflow=0 exact_recv_per_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=69632 expected_recv=130062 expected_overflow=0 exact_recv_per_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=69632 expected_recv=130536 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=69632 expected_recv=127113 actual_valid_recv=128256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=69632 expected_recv=132270 expected_overflow=0 exact_recv_per_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=69632 expected_recv=131443 expected_overflow=0 exact_recv_per_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=69632 expected_recv=130062 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=128256 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=69632 expected_recv=132270 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=69632 expected_recv=131443 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=69632 expected_recv=131013 expected_overflow=0 exact_recv_per_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=69632 expected_recv=130333 expected_overflow=0 exact_recv_per_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=69632 expected_recv=129809 expected_overflow=0 exact_recv_per_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=69632 expected_recv=127885 expected_overflow=0 exact_recv_per_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=69632 expected_recv=131013 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=69632 expected_recv=130333 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=69632 expected_recv=129809 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=69632 expected_recv=127885 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=129024 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=69632 expected_recv=130049 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=69632 expected_recv=127982 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=69632 expected_recv=130049 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=69632 expected_recv=127982 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=128768 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=69632 expected_recv=131763 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=69632 expected_recv=129245 expected_overflow=0 exact_recv_per_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=69632 expected_recv=131763 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=69632 expected_recv=129245 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=130304 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=69632 expected_recv=131802 expected_overflow=0 exact_recv_per_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=69632 expected_recv=128844 expected_overflow=0 exact_recv_per_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=69632 expected_recv=129502 expected_overflow=0 exact_recv_per_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=69632 expected_recv=131802 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=69632 expected_recv=128844 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=69632 expected_recv=129502 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=69632 expected_recv=130150 expected_overflow=0 exact_recv_per_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=69632 expected_recv=131127 expected_overflow=0 exact_recv_per_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=69632 expected_recv=130150 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=69632 expected_recv=131127 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=69632 expected_recv=129611 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=69632 expected_recv=131062 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=69632 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=69632 expected_recv=131662 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=69632 expected_recv=128017 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=69632 expected_recv=129611 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=69632 expected_recv=131062 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=69632 expected_recv=131072 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=69632 expected_recv=131662 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=69632 expected_recv=128017 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=130048 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=69632 expected_recv=126995 expected_overflow=0 exact_recv_per_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=69632 expected_recv=131293 expected_overflow=0 exact_recv_per_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=69632 expected_recv=129758 expected_overflow=0 exact_recv_per_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=69632 expected_recv=126995 actual_valid_recv=128256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=69632 expected_recv=131293 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=69632 expected_recv=129758 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=128256 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=69632 expected_recv=130992 expected_overflow=0 exact_recv_per_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=69632 expected_recv=130992 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=69632 expected_recv=127415 expected_overflow=0 exact_recv_per_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=69632 expected_recv=127415 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=69632 expected_recv=129651 expected_overflow=0 exact_recv_per_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=69632 expected_recv=129591 expected_overflow=0 exact_recv_per_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=129152 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=69632 expected_recv=129651 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=69632 expected_recv=129591 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=69632 expected_recv=129685 expected_overflow=0 exact_recv_per_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131072 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=69632 expected_recv=127189 expected_overflow=0 exact_recv_per_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=69632 expected_recv=131010 expected_overflow=0 exact_recv_per_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=69632 expected_recv=129685 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=69632 expected_recv=127189 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=69632 expected_recv=131010 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=69632 expected_recv=132389 expected_overflow=0 exact_recv_per_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=128640 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=69632 expected_recv=132389 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=69632 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132992 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=69632 expected_recv=131154 expected_overflow=0 exact_recv_per_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=69632 expected_recv=132381 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=69632 expected_recv=131154 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=69632 expected_recv=123220 expected_overflow=0 exact_recv_per_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=69632 expected_recv=123220 actual_valid_recv=124416 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=124416 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=69632 expected_recv=131055 expected_overflow=0 exact_recv_per_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=134272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=69632 expected_recv=131055 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=134272 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=69632 expected_recv=128529 expected_overflow=0 exact_recv_per_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=69632 expected_recv=132381 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=69632 expected_recv=128529 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=69632 expected_recv=130753 expected_overflow=0 exact_recv_per_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=69632 expected_recv=127829 expected_overflow=0 exact_recv_per_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=69632 expected_recv=130753 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=69632 expected_recv=127829 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=129408 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=69632 expected_recv=131930 expected_overflow=0 exact_recv_per_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=69632 expected_recv=131930 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=69632 expected_recv=132388 expected_overflow=0 exact_recv_per_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=69632 expected_recv=131071 expected_overflow=0 exact_recv_per_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=69632 expected_recv=132388 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=69632 expected_recv=131071 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=69632 expected_recv=130846 expected_overflow=0 exact_recv_per_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=69632 expected_recv=126776 expected_overflow=0 exact_recv_per_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=69632 expected_recv=130846 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=69632 expected_recv=126776 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=69632 expected_recv=130345 expected_overflow=0 exact_recv_per_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=128128 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=69632 expected_recv=130345 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=69632 expected_recv=129403 expected_overflow=0 exact_recv_per_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=69632 expected_recv=130990 expected_overflow=0 exact_recv_per_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=69632 expected_recv=129403 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=69632 expected_recv=130990 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=69632 expected_recv=130467 expected_overflow=0 exact_recv_per_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=69632 expected_recv=132280 expected_overflow=0 exact_recv_per_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=69632 expected_recv=130467 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=69632 expected_recv=132280 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=69632 expected_recv=131308 expected_overflow=0 exact_recv_per_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=69632 expected_recv=131308 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=69632 expected_recv=129360 expected_overflow=0 exact_recv_per_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[62128, 71647, 58379, 87598, 65901, 64326, 49477, 64832] avg=65536.0000 max=87598 max_rank=3 min=49477 min_rank=6 imbalance_ratio=1.336639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[62128, 71647, 58379, 87598, 65901, 64326, 49477, 64832] avg=65536.0000 max=87598 max_rank=3 min=49477 min_rank=6 imbalance_ratio=1.336639 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=69632 expected_recv=129360 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[62128, 71647, 58379, 87598, 65901, 64326, 49477, 64832] avg=65536.0000 max=87598 max_rank=3 min=49477 min_rank=6 imbalance_ratio=1.336639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[62128, 71647, 58379, 87598, 65901, 64326, 49477, 64832] avg=65536.0000 max=87598 max_rank=3 min=49477 min_rank=6 imbalance_ratio=1.336639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[62128, 71647, 58379, 87598, 65901, 64326, 49477, 64832] avg=65536.0000 max=87598 max_rank=3 min=49477 min_rank=6 imbalance_ratio=1.336639 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=130688 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[62128, 71647, 58379, 87598, 65901, 64326, 49477, 64832] avg=65536.0000 max=87598 max_rank=3 min=49477 min_rank=6 imbalance_ratio=1.336639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[62128, 71647, 58379, 87598, 65901, 64326, 49477, 64832] avg=65536.0000 max=87598 max_rank=3 min=49477 min_rank=6 imbalance_ratio=1.336639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] avg=65536.0000 max=66192 max_rank=0 min=64326 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21406 max_load_reduction_pct=0.244366 moved_to_replicas=54923 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] avg=65536.0000 max=66192 max_rank=0 min=64326 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21406 max_load_reduction_pct=0.244366 moved_to_replicas=54923 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] avg=65536.0000 max=66192 max_rank=0 min=64326 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21406 max_load_reduction_pct=0.244366 moved_to_replicas=54923 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] avg=65536.0000 max=66192 max_rank=0 min=64326 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21406 max_load_reduction_pct=0.244366 moved_to_replicas=54923 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] avg=65536.0000 max=66192 max_rank=0 min=64326 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21406 max_load_reduction_pct=0.244366 moved_to_replicas=54923 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] avg=65536.0000 max=66192 max_rank=0 min=64326 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21406 max_load_reduction_pct=0.244366 moved_to_replicas=54923 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] avg=65536.0000 max=66192 max_rank=0 min=64326 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21406 max_load_reduction_pct=0.244366 moved_to_replicas=54923 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[62128, 71647, 58379, 87598, 65901, 64326, 49477, 64832] avg=65536.0000 max=87598 max_rank=3 min=49477 min_rank=6 imbalance_ratio=1.336639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[64241, 95983, 52492, 72314, 29522, 91237, 59752, 58747] avg=65536.0000 max=95983 max_rank=1 min=29522 min_rank=4 imbalance_ratio=1.464584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[64241, 95983, 52492, 72314, 29522, 91237, 59752, 58747] avg=65536.0000 max=95983 max_rank=1 min=29522 min_rank=4 imbalance_ratio=1.464584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[64241, 95983, 52492, 72314, 29522, 91237, 59752, 58747] avg=65536.0000 max=95983 max_rank=1 min=29522 min_rank=4 imbalance_ratio=1.464584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[64241, 95983, 52492, 72314, 29522, 91237, 59752, 58747] avg=65536.0000 max=95983 max_rank=1 min=29522 min_rank=4 imbalance_ratio=1.464584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[64241, 95983, 52492, 72314, 29522, 91237, 59752, 58747] avg=65536.0000 max=95983 max_rank=1 min=29522 min_rank=4 imbalance_ratio=1.464584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64461, 66192, 65901, 64326, 66192, 64832] avg=65536.0000 max=66192 max_rank=0 min=64326 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21406 max_load_reduction_pct=0.244366 moved_to_replicas=54923 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[64241, 95983, 52492, 72314, 29522, 91237, 59752, 58747] avg=65536.0000 max=95983 max_rank=1 min=29522 min_rank=4 imbalance_ratio=1.464584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[64241, 95983, 52492, 72314, 29522, 91237, 59752, 58747] avg=65536.0000 max=95983 max_rank=1 min=29522 min_rank=4 imbalance_ratio=1.464584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] avg=65536.0000 max=66192 max_rank=1 min=63213 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29791 max_load_reduction_pct=0.310378 moved_to_replicas=134070 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] avg=65536.0000 max=66192 max_rank=1 min=63213 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29791 max_load_reduction_pct=0.310378 moved_to_replicas=134070 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] avg=65536.0000 max=66192 max_rank=1 min=63213 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29791 max_load_reduction_pct=0.310378 moved_to_replicas=134070 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] avg=65536.0000 max=66192 max_rank=1 min=63213 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29791 max_load_reduction_pct=0.310378 moved_to_replicas=134070 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] avg=65536.0000 max=66192 max_rank=1 min=63213 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29791 max_load_reduction_pct=0.310378 moved_to_replicas=134070 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] avg=65536.0000 max=66192 max_rank=1 min=63213 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29791 max_load_reduction_pct=0.310378 moved_to_replicas=134070 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] avg=65536.0000 max=66192 max_rank=1 min=63213 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29791 max_load_reduction_pct=0.310378 moved_to_replicas=134070 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[64241, 95983, 52492, 72314, 29522, 91237, 59752, 58747] avg=65536.0000 max=95983 max_rank=1 min=29522 min_rank=4 imbalance_ratio=1.464584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[669, 109548, 66562, 97604, 120514, 31069, 65547, 32775] avg=65536.0000 max=120514 max_rank=4 min=669 min_rank=0 imbalance_ratio=1.838898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[669, 109548, 66562, 97604, 120514, 31069, 65547, 32775] avg=65536.0000 max=120514 max_rank=4 min=669 min_rank=0 imbalance_ratio=1.838898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[669, 109548, 66562, 97604, 120514, 31069, 65547, 32775] avg=65536.0000 max=120514 max_rank=4 min=669 min_rank=0 imbalance_ratio=1.838898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[669, 109548, 66562, 97604, 120514, 31069, 65547, 32775] avg=65536.0000 max=120514 max_rank=4 min=669 min_rank=0 imbalance_ratio=1.838898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[669, 109548, 66562, 97604, 120514, 31069, 65547, 32775] avg=65536.0000 max=120514 max_rank=4 min=669 min_rank=0 imbalance_ratio=1.838898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[64241, 66192, 66192, 66191, 63213, 66192, 65875, 66192] avg=65536.0000 max=66192 max_rank=1 min=63213 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29791 max_load_reduction_pct=0.310378 moved_to_replicas=134070 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] avg=65536.0000 max=66193 max_rank=0 min=62243 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=54321 max_load_reduction_pct=0.450744 moved_to_replicas=259556 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[669, 109548, 66562, 97604, 120514, 31069, 65547, 32775] avg=65536.0000 max=120514 max_rank=4 min=669 min_rank=0 imbalance_ratio=1.838898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[669, 109548, 66562, 97604, 120514, 31069, 65547, 32775] avg=65536.0000 max=120514 max_rank=4 min=669 min_rank=0 imbalance_ratio=1.838898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] avg=65536.0000 max=66193 max_rank=0 min=62243 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=54321 max_load_reduction_pct=0.450744 moved_to_replicas=259556 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] avg=65536.0000 max=66193 max_rank=0 min=62243 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=54321 max_load_reduction_pct=0.450744 moved_to_replicas=259556 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] avg=65536.0000 max=66193 max_rank=0 min=62243 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=54321 max_load_reduction_pct=0.450744 moved_to_replicas=259556 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] avg=65536.0000 max=66193 max_rank=0 min=62243 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=54321 max_load_reduction_pct=0.450744 moved_to_replicas=259556 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] avg=65536.0000 max=66193 max_rank=0 min=62243 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=54321 max_load_reduction_pct=0.450744 moved_to_replicas=259556 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] avg=65536.0000 max=66193 max_rank=0 min=62243 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=54321 max_load_reduction_pct=0.450744 moved_to_replicas=259556 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[62384, 109380, 101371, 2852, 97993, 65395, 45160, 39753] avg=65536.0000 max=109380 max_rank=1 min=2852 min_rank=3 imbalance_ratio=1.669006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[669, 109548, 66562, 97604, 120514, 31069, 65547, 32775] avg=65536.0000 max=120514 max_rank=4 min=669 min_rank=0 imbalance_ratio=1.838898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[62384, 109380, 101371, 2852, 97993, 65395, 45160, 39753] avg=65536.0000 max=109380 max_rank=1 min=2852 min_rank=3 imbalance_ratio=1.669006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[62384, 109380, 101371, 2852, 97993, 65395, 45160, 39753] avg=65536.0000 max=109380 max_rank=1 min=2852 min_rank=3 imbalance_ratio=1.669006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[62384, 109380, 101371, 2852, 97993, 65395, 45160, 39753] avg=65536.0000 max=109380 max_rank=1 min=2852 min_rank=3 imbalance_ratio=1.669006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[62384, 109380, 101371, 2852, 97993, 65395, 45160, 39753] avg=65536.0000 max=109380 max_rank=1 min=2852 min_rank=3 imbalance_ratio=1.669006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] avg=65536.0000 max=66193 max_rank=1 min=62384 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43187 max_load_reduction_pct=0.394835 moved_to_replicas=217237 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66191, 62243, 65547, 65539] avg=65536.0000 max=66193 max_rank=0 min=62243 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=54321 max_load_reduction_pct=0.450744 moved_to_replicas=259556 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[62384, 109380, 101371, 2852, 97993, 65395, 45160, 39753] avg=65536.0000 max=109380 max_rank=1 min=2852 min_rank=3 imbalance_ratio=1.669006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[62384, 109380, 101371, 2852, 97993, 65395, 45160, 39753] avg=65536.0000 max=109380 max_rank=1 min=2852 min_rank=3 imbalance_ratio=1.669006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] avg=65536.0000 max=66193 max_rank=1 min=62384 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43187 max_load_reduction_pct=0.394835 moved_to_replicas=217237 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] avg=65536.0000 max=66193 max_rank=1 min=62384 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43187 max_load_reduction_pct=0.394835 moved_to_replicas=217237 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] avg=65536.0000 max=66193 max_rank=1 min=62384 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43187 max_load_reduction_pct=0.394835 moved_to_replicas=217237 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] avg=65536.0000 max=66193 max_rank=1 min=62384 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43187 max_load_reduction_pct=0.394835 moved_to_replicas=217237 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] avg=65536.0000 max=66193 max_rank=1 min=62384 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43187 max_load_reduction_pct=0.394835 moved_to_replicas=217237 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] avg=65536.0000 max=66193 max_rank=1 min=62384 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43187 max_load_reduction_pct=0.394835 moved_to_replicas=217237 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[65391, 68968, 35193, 65798, 60605, 98398, 32858, 97077] avg=65536.0000 max=98398 max_rank=5 min=32858 min_rank=6 imbalance_ratio=1.501434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[62384, 109380, 101371, 2852, 97993, 65395, 45160, 39753] avg=65536.0000 max=109380 max_rank=1 min=2852 min_rank=3 imbalance_ratio=1.669006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[65391, 68968, 35193, 65798, 60605, 98398, 32858, 97077] avg=65536.0000 max=98398 max_rank=5 min=32858 min_rank=6 imbalance_ratio=1.501434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[65391, 68968, 35193, 65798, 60605, 98398, 32858, 97077] avg=65536.0000 max=98398 max_rank=5 min=32858 min_rank=6 imbalance_ratio=1.501434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[65391, 68968, 35193, 65798, 60605, 98398, 32858, 97077] avg=65536.0000 max=98398 max_rank=5 min=32858 min_rank=6 imbalance_ratio=1.501434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[65391, 68968, 35193, 65798, 60605, 98398, 32858, 97077] avg=65536.0000 max=98398 max_rank=5 min=32858 min_rank=6 imbalance_ratio=1.501434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] avg=65536.0000 max=66192 max_rank=1 min=63381 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32206 max_load_reduction_pct=0.327303 moved_to_replicas=130134 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[62384, 66193, 66192, 66190, 66192, 65395, 65551, 66191] avg=65536.0000 max=66193 max_rank=1 min=62384 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43187 max_load_reduction_pct=0.394835 moved_to_replicas=217237 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[65391, 68968, 35193, 65798, 60605, 98398, 32858, 97077] avg=65536.0000 max=98398 max_rank=5 min=32858 min_rank=6 imbalance_ratio=1.501434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[65391, 68968, 35193, 65798, 60605, 98398, 32858, 97077] avg=65536.0000 max=98398 max_rank=5 min=32858 min_rank=6 imbalance_ratio=1.501434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] avg=65536.0000 max=66192 max_rank=1 min=63381 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32206 max_load_reduction_pct=0.327303 moved_to_replicas=130134 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] avg=65536.0000 max=66192 max_rank=1 min=63381 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32206 max_load_reduction_pct=0.327303 moved_to_replicas=130134 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] avg=65536.0000 max=66192 max_rank=1 min=63381 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32206 max_load_reduction_pct=0.327303 moved_to_replicas=130134 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] avg=65536.0000 max=66192 max_rank=1 min=63381 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32206 max_load_reduction_pct=0.327303 moved_to_replicas=130134 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] avg=65536.0000 max=66192 max_rank=1 min=63381 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32206 max_load_reduction_pct=0.327303 moved_to_replicas=130134 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] avg=65536.0000 max=66192 max_rank=1 min=63381 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32206 max_load_reduction_pct=0.327303 moved_to_replicas=130134 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65516, 32600, 32813, 98268, 130700, 98302, 50936, 15153] avg=65536.0000 max=130700 max_rank=4 min=15153 min_rank=7 imbalance_ratio=1.994324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[65391, 68968, 35193, 65798, 60605, 98398, 32858, 97077] avg=65536.0000 max=98398 max_rank=5 min=32858 min_rank=6 imbalance_ratio=1.501434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65516, 32600, 32813, 98268, 130700, 98302, 50936, 15153] avg=65536.0000 max=130700 max_rank=4 min=15153 min_rank=7 imbalance_ratio=1.994324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65516, 32600, 32813, 98268, 130700, 98302, 50936, 15153] avg=65536.0000 max=130700 max_rank=4 min=15153 min_rank=7 imbalance_ratio=1.994324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65516, 32600, 32813, 98268, 130700, 98302, 50936, 15153] avg=65536.0000 max=130700 max_rank=4 min=15153 min_rank=7 imbalance_ratio=1.994324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65516, 32600, 32813, 98268, 130700, 98302, 50936, 15153] avg=65536.0000 max=130700 max_rank=4 min=15153 min_rank=7 imbalance_ratio=1.994324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] avg=65536.0000 max=66192 max_rank=3 min=64342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64508 max_load_reduction_pct=0.493558 moved_to_replicas=256505 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[65391, 66192, 66078, 65798, 63381, 66192, 65064, 66192] avg=65536.0000 max=66192 max_rank=1 min=63381 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32206 max_load_reduction_pct=0.327303 moved_to_replicas=130134 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65516, 32600, 32813, 98268, 130700, 98302, 50936, 15153] avg=65536.0000 max=130700 max_rank=4 min=15153 min_rank=7 imbalance_ratio=1.994324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65516, 32600, 32813, 98268, 130700, 98302, 50936, 15153] avg=65536.0000 max=130700 max_rank=4 min=15153 min_rank=7 imbalance_ratio=1.994324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] avg=65536.0000 max=66192 max_rank=3 min=64342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64508 max_load_reduction_pct=0.493558 moved_to_replicas=256505 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] avg=65536.0000 max=66192 max_rank=3 min=64342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64508 max_load_reduction_pct=0.493558 moved_to_replicas=256505 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] avg=65536.0000 max=66192 max_rank=3 min=64342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64508 max_load_reduction_pct=0.493558 moved_to_replicas=256505 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] avg=65536.0000 max=66192 max_rank=3 min=64342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64508 max_load_reduction_pct=0.493558 moved_to_replicas=256505 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[17467, 97937, 121880, 63920, 131, 59865, 98303, 64785] avg=65536.0000 max=121880 max_rank=2 min=131 min_rank=4 imbalance_ratio=1.859741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] avg=65536.0000 max=66192 max_rank=3 min=64342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64508 max_load_reduction_pct=0.493558 moved_to_replicas=256505 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] avg=65536.0000 max=66192 max_rank=3 min=64342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64508 max_load_reduction_pct=0.493558 moved_to_replicas=256505 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65516, 32600, 32813, 98268, 130700, 98302, 50936, 15153] avg=65536.0000 max=130700 max_rank=4 min=15153 min_rank=7 imbalance_ratio=1.994324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[17467, 97937, 121880, 63920, 131, 59865, 98303, 64785] avg=65536.0000 max=121880 max_rank=2 min=131 min_rank=4 imbalance_ratio=1.859741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[17467, 97937, 121880, 63920, 131, 59865, 98303, 64785] avg=65536.0000 max=121880 max_rank=2 min=131 min_rank=4 imbalance_ratio=1.859741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[17467, 97937, 121880, 63920, 131, 59865, 98303, 64785] avg=65536.0000 max=121880 max_rank=2 min=131 min_rank=4 imbalance_ratio=1.859741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] avg=65536.0000 max=66193 max_rank=0 min=63920 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=55687 max_load_reduction_pct=0.456900 moved_to_replicas=238059 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[17467, 97937, 121880, 63920, 131, 59865, 98303, 64785] avg=65536.0000 max=121880 max_rank=2 min=131 min_rank=4 imbalance_ratio=1.859741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65516, 64342, 64923, 66192, 66192, 66192, 64740, 66191] avg=65536.0000 max=66192 max_rank=3 min=64342 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64508 max_load_reduction_pct=0.493558 moved_to_replicas=256505 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[17467, 97937, 121880, 63920, 131, 59865, 98303, 64785] avg=65536.0000 max=121880 max_rank=2 min=131 min_rank=4 imbalance_ratio=1.859741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[17467, 97937, 121880, 63920, 131, 59865, 98303, 64785] avg=65536.0000 max=121880 max_rank=2 min=131 min_rank=4 imbalance_ratio=1.859741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] avg=65536.0000 max=66193 max_rank=0 min=63920 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=55687 max_load_reduction_pct=0.456900 moved_to_replicas=238059 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] avg=65536.0000 max=66193 max_rank=0 min=63920 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=55687 max_load_reduction_pct=0.456900 moved_to_replicas=238059 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] avg=65536.0000 max=66193 max_rank=0 min=63920 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=55687 max_load_reduction_pct=0.456900 moved_to_replicas=238059 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] avg=65536.0000 max=66193 max_rank=0 min=63920 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=55687 max_load_reduction_pct=0.456900 moved_to_replicas=238059 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[65664, 97975, 130967, 72064, 58765, 502, 32772, 65579] avg=65536.0000 max=130967 max_rank=2 min=502 min_rank=5 imbalance_ratio=1.998398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] avg=65536.0000 max=66193 max_rank=0 min=63920 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=55687 max_load_reduction_pct=0.456900 moved_to_replicas=238059 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] avg=65536.0000 max=66193 max_rank=0 min=63920 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=55687 max_load_reduction_pct=0.456900 moved_to_replicas=238059 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[17467, 97937, 121880, 63920, 131, 59865, 98303, 64785] avg=65536.0000 max=121880 max_rank=2 min=131 min_rank=4 imbalance_ratio=1.859741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[65664, 97975, 130967, 72064, 58765, 502, 32772, 65579] avg=65536.0000 max=130967 max_rank=2 min=502 min_rank=5 imbalance_ratio=1.998398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[65664, 97975, 130967, 72064, 58765, 502, 32772, 65579] avg=65536.0000 max=130967 max_rank=2 min=502 min_rank=5 imbalance_ratio=1.998398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] avg=65536.0000 max=66192 max_rank=1 min=64637 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64775 max_load_reduction_pct=0.494590 moved_to_replicas=206994 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[65664, 97975, 130967, 72064, 58765, 502, 32772, 65579] avg=65536.0000 max=130967 max_rank=2 min=502 min_rank=5 imbalance_ratio=1.998398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[65664, 97975, 130967, 72064, 58765, 502, 32772, 65579] avg=65536.0000 max=130967 max_rank=2 min=502 min_rank=5 imbalance_ratio=1.998398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[66193, 66192, 66191, 63920, 65002, 65813, 66192, 64785] avg=65536.0000 max=66193 max_rank=0 min=63920 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=55687 max_load_reduction_pct=0.456900 moved_to_replicas=238059 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[65664, 97975, 130967, 72064, 58765, 502, 32772, 65579] avg=65536.0000 max=130967 max_rank=2 min=502 min_rank=5 imbalance_ratio=1.998398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] avg=65536.0000 max=66192 max_rank=1 min=64637 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64775 max_load_reduction_pct=0.494590 moved_to_replicas=206994 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[65664, 97975, 130967, 72064, 58765, 502, 32772, 65579] avg=65536.0000 max=130967 max_rank=2 min=502 min_rank=5 imbalance_ratio=1.998398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] avg=65536.0000 max=66192 max_rank=1 min=64637 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64775 max_load_reduction_pct=0.494590 moved_to_replicas=206994 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] avg=65536.0000 max=66192 max_rank=1 min=64637 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64775 max_load_reduction_pct=0.494590 moved_to_replicas=206994 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] avg=65536.0000 max=66192 max_rank=1 min=64637 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64775 max_load_reduction_pct=0.494590 moved_to_replicas=206994 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32781, 36864, 267, 65506, 65534, 141632, 97462, 84242] avg=65536.0000 max=141632 max_rank=5 min=267 min_rank=2 imbalance_ratio=2.161133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] avg=65536.0000 max=66192 max_rank=1 min=64637 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64775 max_load_reduction_pct=0.494590 moved_to_replicas=206994 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] avg=65536.0000 max=66192 max_rank=1 min=64637 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64775 max_load_reduction_pct=0.494590 moved_to_replicas=206994 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[65664, 97975, 130967, 72064, 58765, 502, 32772, 65579] avg=65536.0000 max=130967 max_rank=2 min=502 min_rank=5 imbalance_ratio=1.998398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32781, 36864, 267, 65506, 65534, 141632, 97462, 84242] avg=65536.0000 max=141632 max_rank=5 min=267 min_rank=2 imbalance_ratio=2.161133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] avg=65536.0000 max=66192 max_rank=1 min=63625 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=75440 max_load_reduction_pct=0.532648 moved_to_replicas=249064 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32781, 36864, 267, 65506, 65534, 141632, 97462, 84242] avg=65536.0000 max=141632 max_rank=5 min=267 min_rank=2 imbalance_ratio=2.161133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32781, 36864, 267, 65506, 65534, 141632, 97462, 84242] avg=65536.0000 max=141632 max_rank=5 min=267 min_rank=2 imbalance_ratio=2.161133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32781, 36864, 267, 65506, 65534, 141632, 97462, 84242] avg=65536.0000 max=141632 max_rank=5 min=267 min_rank=2 imbalance_ratio=2.161133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[65664, 66192, 66192, 66192, 64637, 65051, 64781, 65579] avg=65536.0000 max=66192 max_rank=1 min=64637 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64775 max_load_reduction_pct=0.494590 moved_to_replicas=206994 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32781, 36864, 267, 65506, 65534, 141632, 97462, 84242] avg=65536.0000 max=141632 max_rank=5 min=267 min_rank=2 imbalance_ratio=2.161133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] avg=65536.0000 max=66192 max_rank=1 min=63625 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=75440 max_load_reduction_pct=0.532648 moved_to_replicas=249064 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32781, 36864, 267, 65506, 65534, 141632, 97462, 84242] avg=65536.0000 max=141632 max_rank=5 min=267 min_rank=2 imbalance_ratio=2.161133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] avg=65536.0000 max=66192 max_rank=1 min=63625 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=75440 max_load_reduction_pct=0.532648 moved_to_replicas=249064 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] avg=65536.0000 max=66192 max_rank=1 min=63625 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=75440 max_load_reduction_pct=0.532648 moved_to_replicas=249064 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65747, 65526, 32772, 26701, 49145, 87781, 98314] avg=65536.0000 max=98314 max_rank=7 min=26701 min_rank=4 imbalance_ratio=1.500153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] avg=65536.0000 max=66192 max_rank=1 min=63625 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=75440 max_load_reduction_pct=0.532648 moved_to_replicas=249064 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] avg=65536.0000 max=66192 max_rank=1 min=63625 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=75440 max_load_reduction_pct=0.532648 moved_to_replicas=249064 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] avg=65536.0000 max=66192 max_rank=1 min=63625 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=75440 max_load_reduction_pct=0.532648 moved_to_replicas=249064 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32781, 36864, 267, 65506, 65534, 141632, 97462, 84242] avg=65536.0000 max=141632 max_rank=5 min=267 min_rank=2 imbalance_ratio=2.161133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] avg=65536.0000 max=66194 max_rank=5 min=63364 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32120 max_load_reduction_pct=0.326708 moved_to_replicas=169295 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65747, 65526, 32772, 26701, 49145, 87781, 98314] avg=65536.0000 max=98314 max_rank=7 min=26701 min_rank=4 imbalance_ratio=1.500153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65747, 65526, 32772, 26701, 49145, 87781, 98314] avg=65536.0000 max=98314 max_rank=7 min=26701 min_rank=4 imbalance_ratio=1.500153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65747, 65526, 32772, 26701, 49145, 87781, 98314] avg=65536.0000 max=98314 max_rank=7 min=26701 min_rank=4 imbalance_ratio=1.500153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65747, 65526, 32772, 26701, 49145, 87781, 98314] avg=65536.0000 max=98314 max_rank=7 min=26701 min_rank=4 imbalance_ratio=1.500153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[64856, 66192, 63625, 65506, 65534, 66192, 66192, 66191] avg=65536.0000 max=66192 max_rank=1 min=63625 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=75440 max_load_reduction_pct=0.532648 moved_to_replicas=249064 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65747, 65526, 32772, 26701, 49145, 87781, 98314] avg=65536.0000 max=98314 max_rank=7 min=26701 min_rank=4 imbalance_ratio=1.500153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] avg=65536.0000 max=66194 max_rank=5 min=63364 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32120 max_load_reduction_pct=0.326708 moved_to_replicas=169295 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65747, 65526, 32772, 26701, 49145, 87781, 98314] avg=65536.0000 max=98314 max_rank=7 min=26701 min_rank=4 imbalance_ratio=1.500153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] avg=65536.0000 max=66194 max_rank=5 min=63364 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32120 max_load_reduction_pct=0.326708 moved_to_replicas=169295 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[98266, 23925, 32779, 98325, 98290, 68305, 71643, 32755] avg=65536.0000 max=98325 max_rank=3 min=23925 min_rank=1 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] avg=65536.0000 max=66194 max_rank=5 min=63364 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32120 max_load_reduction_pct=0.326708 moved_to_replicas=169295 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] avg=65536.0000 max=66194 max_rank=5 min=63364 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32120 max_load_reduction_pct=0.326708 moved_to_replicas=169295 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] avg=65536.0000 max=66194 max_rank=5 min=63364 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32120 max_load_reduction_pct=0.326708 moved_to_replicas=169295 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] avg=65536.0000 max=66194 max_rank=5 min=63364 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32120 max_load_reduction_pct=0.326708 moved_to_replicas=169295 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65747, 65526, 32772, 26701, 49145, 87781, 98314] avg=65536.0000 max=98314 max_rank=7 min=26701 min_rank=4 imbalance_ratio=1.500153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] avg=65536.0000 max=66192 max_rank=0 min=63623 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=205149 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[98266, 23925, 32779, 98325, 98290, 68305, 71643, 32755] avg=65536.0000 max=98325 max_rank=3 min=23925 min_rank=1 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[98266, 23925, 32779, 98325, 98290, 68305, 71643, 32755] avg=65536.0000 max=98325 max_rank=3 min=23925 min_rank=1 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[98266, 23925, 32779, 98325, 98290, 68305, 71643, 32755] avg=65536.0000 max=98325 max_rank=3 min=23925 min_rank=1 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[98266, 23925, 32779, 98325, 98290, 68305, 71643, 32755] avg=65536.0000 max=98325 max_rank=3 min=23925 min_rank=1 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[98266, 23925, 32779, 98325, 98290, 68305, 71643, 32755] avg=65536.0000 max=98325 max_rank=3 min=23925 min_rank=1 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65747, 65526, 64882, 63364, 66194, 66191, 66192] avg=65536.0000 max=66194 max_rank=5 min=63364 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32120 max_load_reduction_pct=0.326708 moved_to_replicas=169295 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] avg=65536.0000 max=66192 max_rank=0 min=63623 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=205149 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] avg=65536.0000 max=66192 max_rank=0 min=63623 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=205149 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] avg=65536.0000 max=66192 max_rank=0 min=63623 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=205149 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[98266, 23925, 32779, 98325, 98290, 68305, 71643, 32755] avg=65536.0000 max=98325 max_rank=3 min=23925 min_rank=1 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[63999, 32857, 109901, 101805, 4, 150181, 32734, 32807] avg=65536.0000 max=150181 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.291580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] avg=65536.0000 max=66192 max_rank=0 min=63623 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=205149 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] avg=65536.0000 max=66192 max_rank=0 min=63623 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=205149 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] avg=65536.0000 max=66192 max_rank=0 min=63623 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=205149 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] avg=65536.0000 max=66196 max_rank=1 min=63999 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=83985 max_load_reduction_pct=0.559225 moved_to_replicas=328497 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[98266, 23925, 32779, 98325, 98290, 68305, 71643, 32755] avg=65536.0000 max=98325 max_rank=3 min=23925 min_rank=1 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[63999, 32857, 109901, 101805, 4, 150181, 32734, 32807] avg=65536.0000 max=150181 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.291580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[63999, 32857, 109901, 101805, 4, 150181, 32734, 32807] avg=65536.0000 max=150181 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.291580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[63999, 32857, 109901, 101805, 4, 150181, 32734, 32807] avg=65536.0000 max=150181 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.291580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[63999, 32857, 109901, 101805, 4, 150181, 32734, 32807] avg=65536.0000 max=150181 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.291580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[63999, 32857, 109901, 101805, 4, 150181, 32734, 32807] avg=65536.0000 max=150181 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.291580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63623, 64853, 66192, 66192, 66192, 66191, 64853] avg=65536.0000 max=66192 max_rank=0 min=63623 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=205149 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] avg=65536.0000 max=66196 max_rank=1 min=63999 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=83985 max_load_reduction_pct=0.559225 moved_to_replicas=328497 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] avg=65536.0000 max=66196 max_rank=1 min=63999 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=83985 max_load_reduction_pct=0.559225 moved_to_replicas=328497 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] avg=65536.0000 max=66196 max_rank=1 min=63999 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=83985 max_load_reduction_pct=0.559225 moved_to_replicas=328497 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[7773, 32761, 10050, 98273, 46557, 160054, 103295, 65525] avg=65536.0000 max=160054 max_rank=5 min=7773 min_rank=0 imbalance_ratio=2.442230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[63999, 32857, 109901, 101805, 4, 150181, 32734, 32807] avg=65536.0000 max=150181 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.291580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] avg=65536.0000 max=66196 max_rank=1 min=63999 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=83985 max_load_reduction_pct=0.559225 moved_to_replicas=328497 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] avg=65536.0000 max=66196 max_rank=1 min=63999 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=83985 max_load_reduction_pct=0.559225 moved_to_replicas=328497 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] avg=65536.0000 max=66194 max_rank=2 min=61613 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=93860 max_load_reduction_pct=0.586427 moved_to_replicas=327362 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] avg=65536.0000 max=66196 max_rank=1 min=63999 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=83985 max_load_reduction_pct=0.559225 moved_to_replicas=328497 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[63999, 32857, 109901, 101805, 4, 150181, 32734, 32807] avg=65536.0000 max=150181 max_rank=5 min=4 min_rank=4 imbalance_ratio=2.291580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[7773, 32761, 10050, 98273, 46557, 160054, 103295, 65525] avg=65536.0000 max=160054 max_rank=5 min=7773 min_rank=0 imbalance_ratio=2.442230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[7773, 32761, 10050, 98273, 46557, 160054, 103295, 65525] avg=65536.0000 max=160054 max_rank=5 min=7773 min_rank=0 imbalance_ratio=2.442230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[7773, 32761, 10050, 98273, 46557, 160054, 103295, 65525] avg=65536.0000 max=160054 max_rank=5 min=7773 min_rank=0 imbalance_ratio=2.442230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[7773, 32761, 10050, 98273, 46557, 160054, 103295, 65525] avg=65536.0000 max=160054 max_rank=5 min=7773 min_rank=0 imbalance_ratio=2.442230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[7773, 32761, 10050, 98273, 46557, 160054, 103295, 65525] avg=65536.0000 max=160054 max_rank=5 min=7773 min_rank=0 imbalance_ratio=2.442230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[63999, 66196, 66190, 66192, 64445, 66193, 65500, 65573] avg=65536.0000 max=66196 max_rank=1 min=63999 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=83985 max_load_reduction_pct=0.559225 moved_to_replicas=328497 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] avg=65536.0000 max=66194 max_rank=2 min=61613 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=93860 max_load_reduction_pct=0.586427 moved_to_replicas=327362 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[98335, 65465, 77480, 32865, 66, 97474, 98154, 54449] avg=65536.0000 max=98335 max_rank=0 min=66 min_rank=4 imbalance_ratio=1.500473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] avg=65536.0000 max=66194 max_rank=2 min=61613 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=93860 max_load_reduction_pct=0.586427 moved_to_replicas=327362 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] avg=65536.0000 max=66194 max_rank=2 min=61613 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=93860 max_load_reduction_pct=0.586427 moved_to_replicas=327362 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[7773, 32761, 10050, 98273, 46557, 160054, 103295, 65525] avg=65536.0000 max=160054 max_rank=5 min=7773 min_rank=0 imbalance_ratio=2.442230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] avg=65536.0000 max=66194 max_rank=2 min=61613 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=93860 max_load_reduction_pct=0.586427 moved_to_replicas=327362 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] avg=65536.0000 max=66194 max_rank=2 min=61613 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=93860 max_load_reduction_pct=0.586427 moved_to_replicas=327362 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] avg=65536.0000 max=66192 max_rank=0 min=64147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32143 max_load_reduction_pct=0.326872 moved_to_replicas=211190 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] avg=65536.0000 max=66194 max_rank=2 min=61613 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=93860 max_load_reduction_pct=0.586427 moved_to_replicas=327362 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[7773, 32761, 10050, 98273, 46557, 160054, 103295, 65525] avg=65536.0000 max=160054 max_rank=5 min=7773 min_rank=0 imbalance_ratio=2.442230 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[98335, 65465, 77480, 32865, 66, 97474, 98154, 54449] avg=65536.0000 max=98335 max_rank=0 min=66 min_rank=4 imbalance_ratio=1.500473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[98335, 65465, 77480, 32865, 66, 97474, 98154, 54449] avg=65536.0000 max=98335 max_rank=0 min=66 min_rank=4 imbalance_ratio=1.500473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[98335, 65465, 77480, 32865, 66, 97474, 98154, 54449] avg=65536.0000 max=98335 max_rank=0 min=66 min_rank=4 imbalance_ratio=1.500473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[98335, 65465, 77480, 32865, 66, 97474, 98154, 54449] avg=65536.0000 max=98335 max_rank=0 min=66 min_rank=4 imbalance_ratio=1.500473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[98335, 65465, 77480, 32865, 66, 97474, 98154, 54449] avg=65536.0000 max=98335 max_rank=0 min=66 min_rank=4 imbalance_ratio=1.500473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[66191, 61613, 66194, 66192, 66190, 66190, 66193, 65525] avg=65536.0000 max=66194 max_rank=2 min=61613 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=93860 max_load_reduction_pct=0.586427 moved_to_replicas=327362 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] avg=65536.0000 max=66192 max_rank=0 min=64147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32143 max_load_reduction_pct=0.326872 moved_to_replicas=211190 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] avg=65536.0000 max=66192 max_rank=0 min=64147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32143 max_load_reduction_pct=0.326872 moved_to_replicas=211190 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[33484, 32767, 70681, 32776, 106424, 117272, 65536, 65348] avg=65536.0000 max=117272 max_rank=5 min=32767 min_rank=1 imbalance_ratio=1.789429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] avg=65536.0000 max=66192 max_rank=0 min=64147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32143 max_load_reduction_pct=0.326872 moved_to_replicas=211190 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[98335, 65465, 77480, 32865, 66, 97474, 98154, 54449] avg=65536.0000 max=98335 max_rank=0 min=66 min_rank=4 imbalance_ratio=1.500473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] avg=65536.0000 max=66192 max_rank=0 min=64147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32143 max_load_reduction_pct=0.326872 moved_to_replicas=211190 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] avg=65536.0000 max=66192 max_rank=0 min=64147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32143 max_load_reduction_pct=0.326872 moved_to_replicas=211190 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] avg=65536.0000 max=66194 max_rank=4 min=63330 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51078 max_load_reduction_pct=0.435552 moved_to_replicas=191708 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[33484, 32767, 70681, 32776, 106424, 117272, 65536, 65348] avg=65536.0000 max=117272 max_rank=5 min=32767 min_rank=1 imbalance_ratio=1.789429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] avg=65536.0000 max=66192 max_rank=0 min=64147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32143 max_load_reduction_pct=0.326872 moved_to_replicas=211190 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[98335, 65465, 77480, 32865, 66, 97474, 98154, 54449] avg=65536.0000 max=98335 max_rank=0 min=66 min_rank=4 imbalance_ratio=1.500473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[33484, 32767, 70681, 32776, 106424, 117272, 65536, 65348] avg=65536.0000 max=117272 max_rank=5 min=32767 min_rank=1 imbalance_ratio=1.789429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[33484, 32767, 70681, 32776, 106424, 117272, 65536, 65348] avg=65536.0000 max=117272 max_rank=5 min=32767 min_rank=1 imbalance_ratio=1.789429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[33484, 32767, 70681, 32776, 106424, 117272, 65536, 65348] avg=65536.0000 max=117272 max_rank=5 min=32767 min_rank=1 imbalance_ratio=1.789429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[33484, 32767, 70681, 32776, 106424, 117272, 65536, 65348] avg=65536.0000 max=117272 max_rank=5 min=32767 min_rank=1 imbalance_ratio=1.789429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] avg=65536.0000 max=66194 max_rank=4 min=63330 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51078 max_load_reduction_pct=0.435552 moved_to_replicas=191708 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65465, 66191, 64147, 64171, 66192, 66192, 65738] avg=65536.0000 max=66192 max_rank=0 min=64147 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32143 max_load_reduction_pct=0.326872 moved_to_replicas=211190 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] avg=65536.0000 max=66194 max_rank=4 min=63330 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51078 max_load_reduction_pct=0.435552 moved_to_replicas=191708 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[98216, 65412, 65554, 66088, 32771, 32452, 65506, 98289] avg=65536.0000 max=98289 max_rank=7 min=32452 min_rank=5 imbalance_ratio=1.499771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] avg=65536.0000 max=66194 max_rank=4 min=63330 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51078 max_load_reduction_pct=0.435552 moved_to_replicas=191708 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[33484, 32767, 70681, 32776, 106424, 117272, 65536, 65348] avg=65536.0000 max=117272 max_rank=5 min=32767 min_rank=1 imbalance_ratio=1.789429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] avg=65536.0000 max=66194 max_rank=4 min=63330 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51078 max_load_reduction_pct=0.435552 moved_to_replicas=191708 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] avg=65536.0000 max=66194 max_rank=4 min=63330 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51078 max_load_reduction_pct=0.435552 moved_to_replicas=191708 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] avg=65536.0000 max=66192 max_rank=0 min=64549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32097 max_load_reduction_pct=0.326557 moved_to_replicas=127937 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[98216, 65412, 65554, 66088, 32771, 32452, 65506, 98289] avg=65536.0000 max=98289 max_rank=7 min=32452 min_rank=5 imbalance_ratio=1.499771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] avg=65536.0000 max=66194 max_rank=4 min=63330 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51078 max_load_reduction_pct=0.435552 moved_to_replicas=191708 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[98216, 65412, 65554, 66088, 32771, 32452, 65506, 98289] avg=65536.0000 max=98289 max_rank=7 min=32452 min_rank=5 imbalance_ratio=1.499771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[33484, 32767, 70681, 32776, 106424, 117272, 65536, 65348] avg=65536.0000 max=117272 max_rank=5 min=32767 min_rank=1 imbalance_ratio=1.789429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[98216, 65412, 65554, 66088, 32771, 32452, 65506, 98289] avg=65536.0000 max=98289 max_rank=7 min=32452 min_rank=5 imbalance_ratio=1.499771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[98216, 65412, 65554, 66088, 32771, 32452, 65506, 98289] avg=65536.0000 max=98289 max_rank=7 min=32452 min_rank=5 imbalance_ratio=1.499771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[98216, 65412, 65554, 66088, 32771, 32452, 65506, 98289] avg=65536.0000 max=98289 max_rank=7 min=32452 min_rank=5 imbalance_ratio=1.499771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] avg=65536.0000 max=66192 max_rank=0 min=64549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32097 max_load_reduction_pct=0.326557 moved_to_replicas=127937 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] avg=65536.0000 max=66192 max_rank=0 min=64549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32097 max_load_reduction_pct=0.326557 moved_to_replicas=127937 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[66192, 65306, 66190, 63330, 66194, 66192, 65536, 65348] avg=65536.0000 max=66194 max_rank=4 min=63330 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=51078 max_load_reduction_pct=0.435552 moved_to_replicas=191708 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[98216, 65412, 65554, 66088, 32771, 32452, 65506, 98289] avg=65536.0000 max=98289 max_rank=7 min=32452 min_rank=5 imbalance_ratio=1.499771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] avg=65536.0000 max=66192 max_rank=0 min=64549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32097 max_load_reduction_pct=0.326557 moved_to_replicas=127937 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] avg=65536.0000 max=66192 max_rank=0 min=64549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32097 max_load_reduction_pct=0.326557 moved_to_replicas=127937 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] avg=65536.0000 max=66192 max_rank=0 min=64549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32097 max_load_reduction_pct=0.326557 moved_to_replicas=127937 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] avg=65536.0000 max=66192 max_rank=0 min=64549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32097 max_load_reduction_pct=0.326557 moved_to_replicas=127937 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[98216, 65412, 65554, 66088, 32771, 32452, 65506, 98289] avg=65536.0000 max=98289 max_rank=7 min=32452 min_rank=5 imbalance_ratio=1.499771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65412, 65554, 66088, 64795, 64549, 65506, 66192] avg=65536.0000 max=66192 max_rank=0 min=64549 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32097 max_load_reduction_pct=0.326557 moved_to_replicas=127937 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=2102526 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=107776 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=2092027 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=117888 peak_demand=66196 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=2084988 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=123264 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=2099182 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=111104 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=2101494 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=107904 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=2089680 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=121088 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=2102484 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=105472 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=2104835 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=104448 peak_demand=66194 required_capacity=1.010040 + [2026-08-30 05:15:28] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 524.9 | throughput per GPU (TFLOP/s/GPU): 635.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.996042E+00 | loss scale: 1.0 | grad norm: 5.229 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=69632 expected_recv=126038 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=69632 expected_recv=129009 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=69632 expected_recv=131609 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=69632 expected_recv=126038 actual_valid_recv=127616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=69632 expected_recv=129009 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=127616 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=69632 expected_recv=131609 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=130176 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=139264 valid_num_tokens=132736 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=69632 expected_recv=131270 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=69632 expected_recv=131270 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=69632 expected_recv=127931 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=69632 expected_recv=128216 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=69632 expected_recv=127931 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=69632 expected_recv=128216 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=69632 expected_recv=131624 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=129280 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=69632 expected_recv=131624 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=69632 expected_recv=128361 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=69632 expected_recv=128361 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=134144 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=69632 expected_recv=127248 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=69632 expected_recv=131047 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=69632 expected_recv=127248 actual_valid_recv=128896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=69632 expected_recv=131047 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=128896 slack_slots=10368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=69632 expected_recv=129867 expected_overflow=0 exact_recv_per_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=69632 expected_recv=129380 expected_overflow=0 exact_recv_per_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=69632 expected_recv=129867 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=69632 expected_recv=129380 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=69632 expected_recv=129722 expected_overflow=0 exact_recv_per_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=69632 expected_recv=130071 expected_overflow=0 exact_recv_per_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=69632 expected_recv=129722 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=69632 expected_recv=130071 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=69632 expected_recv=130759 expected_overflow=0 exact_recv_per_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=69632 expected_recv=130759 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=69632 expected_recv=129024 expected_overflow=0 exact_recv_per_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=69632 expected_recv=129024 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=69632 expected_recv=129602 expected_overflow=0 exact_recv_per_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=69632 expected_recv=131109 expected_overflow=0 exact_recv_per_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=69632 expected_recv=130931 expected_overflow=0 exact_recv_per_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=69632 expected_recv=131109 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=69632 expected_recv=129602 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=69632 expected_recv=130931 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=69632 expected_recv=130897 expected_overflow=0 exact_recv_per_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=69632 expected_recv=130897 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=69632 expected_recv=127218 expected_overflow=0 exact_recv_per_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=69632 expected_recv=127218 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=69632 expected_recv=128540 expected_overflow=0 exact_recv_per_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=129024 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=69632 expected_recv=128540 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=129920 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=69632 expected_recv=130380 expected_overflow=0 exact_recv_per_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=134016 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=69632 expected_recv=130380 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=69632 expected_recv=130603 expected_overflow=0 exact_recv_per_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=69632 expected_recv=126839 expected_overflow=0 exact_recv_per_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=69632 expected_recv=130603 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=69632 expected_recv=126839 actual_valid_recv=127872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=69632 expected_recv=131216 expected_overflow=0 exact_recv_per_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=127872 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=69632 expected_recv=131079 expected_overflow=0 exact_recv_per_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=69632 expected_recv=131216 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=69632 expected_recv=131079 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=69632 expected_recv=127482 expected_overflow=0 exact_recv_per_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=69632 expected_recv=127482 actual_valid_recv=128896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=69632 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=69632 expected_recv=130259 expected_overflow=0 exact_recv_per_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=128896 slack_slots=10368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=69632 expected_recv=132382 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=69632 expected_recv=130259 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=69632 expected_recv=130220 expected_overflow=0 exact_recv_per_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=69632 expected_recv=125440 expected_overflow=0 exact_recv_per_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=69632 expected_recv=130220 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=69632 expected_recv=125440 actual_valid_recv=126848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=131584 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=69632 expected_recv=131383 expected_overflow=0 exact_recv_per_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=126848 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=69632 expected_recv=131383 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=69632 expected_recv=132388 expected_overflow=0 exact_recv_per_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=69632 expected_recv=132388 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=69632 expected_recv=131049 expected_overflow=0 exact_recv_per_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=69632 expected_recv=131161 expected_overflow=0 exact_recv_per_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=69632 expected_recv=131049 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=69632 expected_recv=131161 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=69632 expected_recv=129793 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=69632 expected_recv=129793 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=131328 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=69632 expected_recv=131459 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=69632 expected_recv=125403 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=69632 expected_recv=131459 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=69632 expected_recv=130301 expected_overflow=0 exact_recv_per_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=69632 expected_recv=125403 actual_valid_recv=127104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=69632 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=69632 expected_recv=130301 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=127104 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=69632 expected_recv=132385 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=69632 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132224 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=139264 valid_num_tokens=134144 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=69632 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=69632 expected_recv=130053 expected_overflow=0 exact_recv_per_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=69632 expected_recv=130053 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=69632 expected_recv=127593 expected_overflow=0 exact_recv_per_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=69632 expected_recv=127593 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=69632 expected_recv=131465 expected_overflow=0 exact_recv_per_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=69632 expected_recv=131562 expected_overflow=0 exact_recv_per_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=128768 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=69632 expected_recv=132013 expected_overflow=0 exact_recv_per_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=69632 expected_recv=131465 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=69632 expected_recv=131562 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=69632 expected_recv=129346 expected_overflow=0 exact_recv_per_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=69632 expected_recv=132013 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=69632 expected_recv=129346 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=69632 expected_recv=129019 expected_overflow=0 exact_recv_per_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130944 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=69632 expected_recv=129019 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=130432 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=69632 expected_recv=131071 expected_overflow=0 exact_recv_per_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=69632 expected_recv=130426 expected_overflow=0 exact_recv_per_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=69632 expected_recv=128121 expected_overflow=0 exact_recv_per_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133888 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=69632 expected_recv=131071 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=69632 expected_recv=130426 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=69632 expected_recv=128121 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=129152 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=69632 expected_recv=130363 expected_overflow=0 exact_recv_per_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=69632 expected_recv=130363 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=69632 expected_recv=131341 expected_overflow=0 exact_recv_per_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131712 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=69632 expected_recv=131341 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132480 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=69632 expected_recv=131406 expected_overflow=0 exact_recv_per_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=69632 expected_recv=130732 expected_overflow=0 exact_recv_per_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=69632 expected_recv=131406 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=69632 expected_recv=130732 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=69632 expected_recv=130193 expected_overflow=0 exact_recv_per_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=133248 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=132608 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=69632 expected_recv=129392 expected_overflow=0 exact_recv_per_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=69632 expected_recv=130193 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=69632 expected_recv=129392 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=139264 valid_num_tokens=131968 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=131200 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133120 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=69632 expected_recv=126965 expected_overflow=0 exact_recv_per_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=69632 expected_recv=130174 expected_overflow=0 exact_recv_per_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=69632 expected_recv=126965 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=69632 expected_recv=131951 expected_overflow=0 exact_recv_per_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=69632 expected_recv=130342 expected_overflow=0 exact_recv_per_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=128768 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=69632 expected_recv=130174 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133376 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=69632 expected_recv=130302 expected_overflow=0 exact_recv_per_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=69632 expected_recv=131951 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=69632 expected_recv=130342 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131456 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=69632 expected_recv=130302 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=69632 expected_recv=129573 expected_overflow=0 exact_recv_per_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=131840 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=69632 expected_recv=129573 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=130816 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=69632 expected_recv=131033 expected_overflow=0 exact_recv_per_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=69632 expected_recv=131033 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=69632 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133760 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=69632 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=69632 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=139264 valid_num_tokens=133504 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=69632 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=69632 expected_recv=124703 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=69632 expected_recv=131181 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=133632 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=69632 expected_recv=131088 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=69632 expected_recv=124703 actual_valid_recv=126336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=69632 expected_recv=131181 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=69632 expected_recv=131088 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=126336 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132352 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132096 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[67952, 80163, 61741, 77204, 65417, 59521, 52431, 59859] avg=65536.0000 max=80163 max_rank=1 min=52431 min_rank=6 imbalance_ratio=1.223190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[67952, 80163, 61741, 77204, 65417, 59521, 52431, 59859] avg=65536.0000 max=80163 max_rank=1 min=52431 min_rank=6 imbalance_ratio=1.223190 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=69632 expected_recv=132066 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[67952, 80163, 61741, 77204, 65417, 59521, 52431, 59859] avg=65536.0000 max=80163 max_rank=1 min=52431 min_rank=6 imbalance_ratio=1.223190 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=69632 expected_recv=132066 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=139264 valid_num_tokens=132864 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[67952, 80163, 61741, 77204, 65417, 59521, 52431, 59859] avg=65536.0000 max=80163 max_rank=1 min=52431 min_rank=6 imbalance_ratio=1.223190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] avg=65536.0000 max=66192 max_rank=0 min=63501 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13971 max_load_reduction_pct=0.174282 moved_to_replicas=57650 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] avg=65536.0000 max=66192 max_rank=0 min=63501 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13971 max_load_reduction_pct=0.174282 moved_to_replicas=57650 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[67952, 80163, 61741, 77204, 65417, 59521, 52431, 59859] avg=65536.0000 max=80163 max_rank=1 min=52431 min_rank=6 imbalance_ratio=1.223190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[67952, 80163, 61741, 77204, 65417, 59521, 52431, 59859] avg=65536.0000 max=80163 max_rank=1 min=52431 min_rank=6 imbalance_ratio=1.223190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] avg=65536.0000 max=66192 max_rank=0 min=63501 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13971 max_load_reduction_pct=0.174282 moved_to_replicas=57650 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[67952, 80163, 61741, 77204, 65417, 59521, 52431, 59859] avg=65536.0000 max=80163 max_rank=1 min=52431 min_rank=6 imbalance_ratio=1.223190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] avg=65536.0000 max=66192 max_rank=0 min=63501 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13971 max_load_reduction_pct=0.174282 moved_to_replicas=57650 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] avg=65536.0000 max=66192 max_rank=0 min=63501 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13971 max_load_reduction_pct=0.174282 moved_to_replicas=57650 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[67952, 80163, 61741, 77204, 65417, 59521, 52431, 59859] avg=65536.0000 max=80163 max_rank=1 min=52431 min_rank=6 imbalance_ratio=1.223190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[42807, 111377, 69350, 71103, 33654, 73800, 58161, 64036] avg=65536.0000 max=111377 max_rank=1 min=33654 min_rank=4 imbalance_ratio=1.699478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] avg=65536.0000 max=66192 max_rank=0 min=63501 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13971 max_load_reduction_pct=0.174282 moved_to_replicas=57650 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[42807, 111377, 69350, 71103, 33654, 73800, 58161, 64036] avg=65536.0000 max=111377 max_rank=1 min=33654 min_rank=4 imbalance_ratio=1.699478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] avg=65536.0000 max=66192 max_rank=0 min=63501 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13971 max_load_reduction_pct=0.174282 moved_to_replicas=57650 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[42807, 111377, 69350, 71103, 33654, 73800, 58161, 64036] avg=65536.0000 max=111377 max_rank=1 min=33654 min_rank=4 imbalance_ratio=1.699478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[42807, 111377, 69350, 71103, 33654, 73800, 58161, 64036] avg=65536.0000 max=111377 max_rank=1 min=33654 min_rank=4 imbalance_ratio=1.699478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] avg=65536.0000 max=66193 max_rank=0 min=63100 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=45184 max_load_reduction_pct=0.405685 moved_to_replicas=118822 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63501, 66192, 65417, 66192, 66192, 64410] avg=65536.0000 max=66192 max_rank=0 min=63501 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=13971 max_load_reduction_pct=0.174282 moved_to_replicas=57650 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] avg=65536.0000 max=66193 max_rank=0 min=63100 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=45184 max_load_reduction_pct=0.405685 moved_to_replicas=118822 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] avg=65536.0000 max=66193 max_rank=0 min=63100 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=45184 max_load_reduction_pct=0.405685 moved_to_replicas=118822 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[42807, 111377, 69350, 71103, 33654, 73800, 58161, 64036] avg=65536.0000 max=111377 max_rank=1 min=33654 min_rank=4 imbalance_ratio=1.699478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[42807, 111377, 69350, 71103, 33654, 73800, 58161, 64036] avg=65536.0000 max=111377 max_rank=1 min=33654 min_rank=4 imbalance_ratio=1.699478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] avg=65536.0000 max=66193 max_rank=0 min=63100 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=45184 max_load_reduction_pct=0.405685 moved_to_replicas=118822 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[42807, 111377, 69350, 71103, 33654, 73800, 58161, 64036] avg=65536.0000 max=111377 max_rank=1 min=33654 min_rank=4 imbalance_ratio=1.699478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32512, 98173, 79808, 53399, 104797, 31529, 65505, 58565] avg=65536.0000 max=104797 max_rank=4 min=31529 min_rank=5 imbalance_ratio=1.599075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] avg=65536.0000 max=66193 max_rank=0 min=63100 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=45184 max_load_reduction_pct=0.405685 moved_to_replicas=118822 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[42807, 111377, 69350, 71103, 33654, 73800, 58161, 64036] avg=65536.0000 max=111377 max_rank=1 min=33654 min_rank=4 imbalance_ratio=1.699478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32512, 98173, 79808, 53399, 104797, 31529, 65505, 58565] avg=65536.0000 max=104797 max_rank=4 min=31529 min_rank=5 imbalance_ratio=1.599075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] avg=65536.0000 max=66193 max_rank=0 min=63100 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=45184 max_load_reduction_pct=0.405685 moved_to_replicas=118822 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] avg=65536.0000 max=66193 max_rank=0 min=63100 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=45184 max_load_reduction_pct=0.405685 moved_to_replicas=118822 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32512, 98173, 79808, 53399, 104797, 31529, 65505, 58565] avg=65536.0000 max=104797 max_rank=4 min=31529 min_rank=5 imbalance_ratio=1.599075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32512, 98173, 79808, 53399, 104797, 31529, 65505, 58565] avg=65536.0000 max=104797 max_rank=4 min=31529 min_rank=5 imbalance_ratio=1.599075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] avg=65536.0000 max=66193 max_rank=3 min=63606 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=38604 max_load_reduction_pct=0.368369 moved_to_replicas=168192 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] avg=65536.0000 max=66193 max_rank=3 min=63606 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=38604 max_load_reduction_pct=0.368369 moved_to_replicas=168192 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[66193, 66191, 66192, 66192, 66192, 66192, 63100, 64036] avg=65536.0000 max=66193 max_rank=0 min=63100 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=45184 max_load_reduction_pct=0.405685 moved_to_replicas=118822 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] avg=65536.0000 max=66193 max_rank=3 min=63606 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=38604 max_load_reduction_pct=0.368369 moved_to_replicas=168192 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32512, 98173, 79808, 53399, 104797, 31529, 65505, 58565] avg=65536.0000 max=104797 max_rank=4 min=31529 min_rank=5 imbalance_ratio=1.599075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] avg=65536.0000 max=66193 max_rank=3 min=63606 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=38604 max_load_reduction_pct=0.368369 moved_to_replicas=168192 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32512, 98173, 79808, 53399, 104797, 31529, 65505, 58565] avg=65536.0000 max=104797 max_rank=4 min=31529 min_rank=5 imbalance_ratio=1.599075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32512, 98173, 79808, 53399, 104797, 31529, 65505, 58565] avg=65536.0000 max=104797 max_rank=4 min=31529 min_rank=5 imbalance_ratio=1.599075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32816, 67346, 130519, 16, 98303, 66675, 63726, 64887] avg=65536.0000 max=130519 max_rank=2 min=16 min_rank=3 imbalance_ratio=1.991562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] avg=65536.0000 max=66193 max_rank=3 min=63606 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=38604 max_load_reduction_pct=0.368369 moved_to_replicas=168192 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32816, 67346, 130519, 16, 98303, 66675, 63726, 64887] avg=65536.0000 max=130519 max_rank=2 min=16 min_rank=3 imbalance_ratio=1.991562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] avg=65536.0000 max=66193 max_rank=3 min=63606 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=38604 max_load_reduction_pct=0.368369 moved_to_replicas=168192 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32512, 98173, 79808, 53399, 104797, 31529, 65505, 58565] avg=65536.0000 max=104797 max_rank=4 min=31529 min_rank=5 imbalance_ratio=1.599075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] avg=65536.0000 max=66193 max_rank=3 min=63606 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=38604 max_load_reduction_pct=0.368369 moved_to_replicas=168192 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32816, 67346, 130519, 16, 98303, 66675, 63726, 64887] avg=65536.0000 max=130519 max_rank=2 min=16 min_rank=3 imbalance_ratio=1.991562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32816, 67346, 130519, 16, 98303, 66675, 63726, 64887] avg=65536.0000 max=130519 max_rank=2 min=16 min_rank=3 imbalance_ratio=1.991562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] avg=65536.0000 max=66192 max_rank=1 min=64826 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64327 max_load_reduction_pct=0.492855 moved_to_replicas=196559 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] avg=65536.0000 max=66192 max_rank=1 min=64826 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64327 max_load_reduction_pct=0.492855 moved_to_replicas=196559 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 66193, 66192, 64217, 65505, 63606] avg=65536.0000 max=66193 max_rank=3 min=63606 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=38604 max_load_reduction_pct=0.368369 moved_to_replicas=168192 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] avg=65536.0000 max=66192 max_rank=1 min=64826 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64327 max_load_reduction_pct=0.492855 moved_to_replicas=196559 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] avg=65536.0000 max=66192 max_rank=1 min=64826 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64327 max_load_reduction_pct=0.492855 moved_to_replicas=196559 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32816, 67346, 130519, 16, 98303, 66675, 63726, 64887] avg=65536.0000 max=130519 max_rank=2 min=16 min_rank=3 imbalance_ratio=1.991562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32816, 67346, 130519, 16, 98303, 66675, 63726, 64887] avg=65536.0000 max=130519 max_rank=2 min=16 min_rank=3 imbalance_ratio=1.991562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32816, 67346, 130519, 16, 98303, 66675, 63726, 64887] avg=65536.0000 max=130519 max_rank=2 min=16 min_rank=3 imbalance_ratio=1.991562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[29992, 65326, 35515, 130637, 33049, 81310, 65587, 82872] avg=65536.0000 max=130637 max_rank=3 min=29992 min_rank=0 imbalance_ratio=1.993362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[29992, 65326, 35515, 130637, 33049, 81310, 65587, 82872] avg=65536.0000 max=130637 max_rank=3 min=29992 min_rank=0 imbalance_ratio=1.993362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] avg=65536.0000 max=66192 max_rank=1 min=64826 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64327 max_load_reduction_pct=0.492855 moved_to_replicas=196559 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] avg=65536.0000 max=66192 max_rank=1 min=64826 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64327 max_load_reduction_pct=0.492855 moved_to_replicas=196559 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32816, 67346, 130519, 16, 98303, 66675, 63726, 64887] avg=65536.0000 max=130519 max_rank=2 min=16 min_rank=3 imbalance_ratio=1.991562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] avg=65536.0000 max=66192 max_rank=1 min=64826 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64327 max_load_reduction_pct=0.492855 moved_to_replicas=196559 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[29992, 65326, 35515, 130637, 33049, 81310, 65587, 82872] avg=65536.0000 max=130637 max_rank=3 min=29992 min_rank=0 imbalance_ratio=1.993362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[29992, 65326, 35515, 130637, 33049, 81310, 65587, 82872] avg=65536.0000 max=130637 max_rank=3 min=29992 min_rank=0 imbalance_ratio=1.993362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] avg=65536.0000 max=66192 max_rank=2 min=63869 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64445 max_load_reduction_pct=0.493314 moved_to_replicas=197073 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] avg=65536.0000 max=66192 max_rank=2 min=63869 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64445 max_load_reduction_pct=0.493314 moved_to_replicas=197073 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[64927, 66192, 66192, 64826, 66192, 66192, 64880, 64887] avg=65536.0000 max=66192 max_rank=1 min=64826 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64327 max_load_reduction_pct=0.492855 moved_to_replicas=196559 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] avg=65536.0000 max=66192 max_rank=2 min=63869 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64445 max_load_reduction_pct=0.493314 moved_to_replicas=197073 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] avg=65536.0000 max=66192 max_rank=2 min=63869 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64445 max_load_reduction_pct=0.493314 moved_to_replicas=197073 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[29992, 65326, 35515, 130637, 33049, 81310, 65587, 82872] avg=65536.0000 max=130637 max_rank=3 min=29992 min_rank=0 imbalance_ratio=1.993362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[29992, 65326, 35515, 130637, 33049, 81310, 65587, 82872] avg=65536.0000 max=130637 max_rank=3 min=29992 min_rank=0 imbalance_ratio=1.993362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[29992, 65326, 35515, 130637, 33049, 81310, 65587, 82872] avg=65536.0000 max=130637 max_rank=3 min=29992 min_rank=0 imbalance_ratio=1.993362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65562, 37793, 54385, 66603, 136829, 82366, 31128, 49622] avg=65536.0000 max=136829 max_rank=4 min=31128 min_rank=6 imbalance_ratio=2.087845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65562, 37793, 54385, 66603, 136829, 82366, 31128, 49622] avg=65536.0000 max=136829 max_rank=4 min=31128 min_rank=6 imbalance_ratio=2.087845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] avg=65536.0000 max=66192 max_rank=2 min=63869 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64445 max_load_reduction_pct=0.493314 moved_to_replicas=197073 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] avg=65536.0000 max=66192 max_rank=2 min=63869 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64445 max_load_reduction_pct=0.493314 moved_to_replicas=197073 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[29992, 65326, 35515, 130637, 33049, 81310, 65587, 82872] avg=65536.0000 max=130637 max_rank=3 min=29992 min_rank=0 imbalance_ratio=1.993362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] avg=65536.0000 max=66192 max_rank=2 min=63869 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64445 max_load_reduction_pct=0.493314 moved_to_replicas=197073 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65562, 37793, 54385, 66603, 136829, 82366, 31128, 49622] avg=65536.0000 max=136829 max_rank=4 min=31128 min_rank=6 imbalance_ratio=2.087845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65562, 37793, 54385, 66603, 136829, 82366, 31128, 49622] avg=65536.0000 max=136829 max_rank=4 min=31128 min_rank=6 imbalance_ratio=2.087845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] avg=65536.0000 max=66193 max_rank=2 min=63874 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=70636 max_load_reduction_pct=0.516236 moved_to_replicas=176499 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] avg=65536.0000 max=66193 max_rank=2 min=63874 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=70636 max_load_reduction_pct=0.516236 moved_to_replicas=176499 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[63869, 65326, 66192, 66192, 64738, 66192, 65587, 66192] avg=65536.0000 max=66192 max_rank=2 min=63869 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64445 max_load_reduction_pct=0.493314 moved_to_replicas=197073 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] avg=65536.0000 max=66193 max_rank=2 min=63874 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=70636 max_load_reduction_pct=0.516236 moved_to_replicas=176499 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] avg=65536.0000 max=66193 max_rank=2 min=63874 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=70636 max_load_reduction_pct=0.516236 moved_to_replicas=176499 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65562, 37793, 54385, 66603, 136829, 82366, 31128, 49622] avg=65536.0000 max=136829 max_rank=4 min=31128 min_rank=6 imbalance_ratio=2.087845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65562, 37793, 54385, 66603, 136829, 82366, 31128, 49622] avg=65536.0000 max=136829 max_rank=4 min=31128 min_rank=6 imbalance_ratio=2.087845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65562, 37793, 54385, 66603, 136829, 82366, 31128, 49622] avg=65536.0000 max=136829 max_rank=4 min=31128 min_rank=6 imbalance_ratio=2.087845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[271, 122428, 65743, 98152, 27149, 65163, 86870, 58512] avg=65536.0000 max=122428 max_rank=1 min=271 min_rank=0 imbalance_ratio=1.868103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[271, 122428, 65743, 98152, 27149, 65163, 86870, 58512] avg=65536.0000 max=122428 max_rank=1 min=271 min_rank=0 imbalance_ratio=1.868103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] avg=65536.0000 max=66193 max_rank=2 min=63874 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=70636 max_load_reduction_pct=0.516236 moved_to_replicas=176499 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] avg=65536.0000 max=66193 max_rank=2 min=63874 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=70636 max_load_reduction_pct=0.516236 moved_to_replicas=176499 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] avg=65536.0000 max=66193 max_rank=2 min=63874 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=70636 max_load_reduction_pct=0.516236 moved_to_replicas=176499 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65562, 37793, 54385, 66603, 136829, 82366, 31128, 49622] avg=65536.0000 max=136829 max_rank=4 min=31128 min_rank=6 imbalance_ratio=2.087845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[271, 122428, 65743, 98152, 27149, 65163, 86870, 58512] avg=65536.0000 max=122428 max_rank=1 min=271 min_rank=0 imbalance_ratio=1.868103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[271, 122428, 65743, 98152, 27149, 65163, 86870, 58512] avg=65536.0000 max=122428 max_rank=1 min=271 min_rank=0 imbalance_ratio=1.868103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] avg=65536.0000 max=66194 max_rank=6 min=63617 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=56234 max_load_reduction_pct=0.459323 moved_to_replicas=217167 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] avg=65536.0000 max=66194 max_rank=6 min=63617 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=56234 max_load_reduction_pct=0.459323 moved_to_replicas=217167 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65562, 66191, 66193, 66192, 66193, 66193, 63890, 63874] avg=65536.0000 max=66193 max_rank=2 min=63874 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=70636 max_load_reduction_pct=0.516236 moved_to_replicas=176499 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[271, 122428, 65743, 98152, 27149, 65163, 86870, 58512] avg=65536.0000 max=122428 max_rank=1 min=271 min_rank=0 imbalance_ratio=1.868103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[271, 122428, 65743, 98152, 27149, 65163, 86870, 58512] avg=65536.0000 max=122428 max_rank=1 min=271 min_rank=0 imbalance_ratio=1.868103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[271, 122428, 65743, 98152, 27149, 65163, 86870, 58512] avg=65536.0000 max=122428 max_rank=1 min=271 min_rank=0 imbalance_ratio=1.868103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] avg=65536.0000 max=66194 max_rank=6 min=63617 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=56234 max_load_reduction_pct=0.459323 moved_to_replicas=217167 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] avg=65536.0000 max=66194 max_rank=6 min=63617 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=56234 max_load_reduction_pct=0.459323 moved_to_replicas=217167 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[95725, 65603, 130887, 33302, 38420, 32773, 32776, 94802] avg=65536.0000 max=130887 max_rank=2 min=32773 min_rank=5 imbalance_ratio=1.997177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[95725, 65603, 130887, 33302, 38420, 32773, 32776, 94802] avg=65536.0000 max=130887 max_rank=2 min=32773 min_rank=5 imbalance_ratio=1.997177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] avg=65536.0000 max=66194 max_rank=6 min=63617 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=56234 max_load_reduction_pct=0.459323 moved_to_replicas=217167 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] avg=65536.0000 max=66194 max_rank=6 min=63617 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=56234 max_load_reduction_pct=0.459323 moved_to_replicas=217167 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] avg=65536.0000 max=66194 max_rank=6 min=63617 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=56234 max_load_reduction_pct=0.459323 moved_to_replicas=217167 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[271, 122428, 65743, 98152, 27149, 65163, 86870, 58512] avg=65536.0000 max=122428 max_rank=1 min=271 min_rank=0 imbalance_ratio=1.868103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] avg=65536.0000 max=66194 max_rank=4 min=63673 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64693 max_load_reduction_pct=0.494266 moved_to_replicas=244891 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[95725, 65603, 130887, 33302, 38420, 32773, 32776, 94802] avg=65536.0000 max=130887 max_rank=2 min=32773 min_rank=5 imbalance_ratio=1.997177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] avg=65536.0000 max=66194 max_rank=4 min=63673 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64693 max_load_reduction_pct=0.494266 moved_to_replicas=244891 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[95725, 65603, 130887, 33302, 38420, 32773, 32776, 94802] avg=65536.0000 max=130887 max_rank=2 min=32773 min_rank=5 imbalance_ratio=1.997177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[64995, 66192, 65743, 66192, 66192, 65163, 66194, 63617] avg=65536.0000 max=66194 max_rank=6 min=63617 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=56234 max_load_reduction_pct=0.459323 moved_to_replicas=217167 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[95725, 65603, 130887, 33302, 38420, 32773, 32776, 94802] avg=65536.0000 max=130887 max_rank=2 min=32773 min_rank=5 imbalance_ratio=1.997177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[95725, 65603, 130887, 33302, 38420, 32773, 32776, 94802] avg=65536.0000 max=130887 max_rank=2 min=32773 min_rank=5 imbalance_ratio=1.997177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[95725, 65603, 130887, 33302, 38420, 32773, 32776, 94802] avg=65536.0000 max=130887 max_rank=2 min=32773 min_rank=5 imbalance_ratio=1.997177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] avg=65536.0000 max=66194 max_rank=4 min=63673 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64693 max_load_reduction_pct=0.494266 moved_to_replicas=244891 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] avg=65536.0000 max=66194 max_rank=4 min=63673 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64693 max_load_reduction_pct=0.494266 moved_to_replicas=244891 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32904, 18498, 98253, 94014, 65526, 109957, 32909, 72227] avg=65536.0000 max=109957 max_rank=5 min=18498 min_rank=1 imbalance_ratio=1.677811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32904, 18498, 98253, 94014, 65526, 109957, 32909, 72227] avg=65536.0000 max=109957 max_rank=5 min=18498 min_rank=1 imbalance_ratio=1.677811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] avg=65536.0000 max=66194 max_rank=4 min=63673 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64693 max_load_reduction_pct=0.494266 moved_to_replicas=244891 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] avg=65536.0000 max=66194 max_rank=4 min=63673 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64693 max_load_reduction_pct=0.494266 moved_to_replicas=244891 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] avg=65536.0000 max=66194 max_rank=4 min=63673 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64693 max_load_reduction_pct=0.494266 moved_to_replicas=244891 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[95725, 65603, 130887, 33302, 38420, 32773, 32776, 94802] avg=65536.0000 max=130887 max_rank=2 min=32773 min_rank=5 imbalance_ratio=1.997177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] avg=65536.0000 max=66194 max_rank=5 min=62831 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=43763 max_load_reduction_pct=0.398001 moved_to_replicas=217838 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32904, 18498, 98253, 94014, 65526, 109957, 32909, 72227] avg=65536.0000 max=109957 max_rank=5 min=18498 min_rank=1 imbalance_ratio=1.677811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] avg=65536.0000 max=66194 max_rank=5 min=62831 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=43763 max_load_reduction_pct=0.398001 moved_to_replicas=217838 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32904, 18498, 98253, 94014, 65526, 109957, 32909, 72227] avg=65536.0000 max=109957 max_rank=5 min=18498 min_rank=1 imbalance_ratio=1.677811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[66192, 65603, 66192, 63673, 66194, 65536, 64708, 66190] avg=65536.0000 max=66194 max_rank=4 min=63673 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64693 max_load_reduction_pct=0.494266 moved_to_replicas=244891 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32904, 18498, 98253, 94014, 65526, 109957, 32909, 72227] avg=65536.0000 max=109957 max_rank=5 min=18498 min_rank=1 imbalance_ratio=1.677811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32904, 18498, 98253, 94014, 65526, 109957, 32909, 72227] avg=65536.0000 max=109957 max_rank=5 min=18498 min_rank=1 imbalance_ratio=1.677811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32904, 18498, 98253, 94014, 65526, 109957, 32909, 72227] avg=65536.0000 max=109957 max_rank=5 min=18498 min_rank=1 imbalance_ratio=1.677811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] avg=65536.0000 max=66194 max_rank=5 min=62831 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=43763 max_load_reduction_pct=0.398001 moved_to_replicas=217838 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[98348, 98121, 44536, 32781, 21870, 33412, 96899, 98321] avg=65536.0000 max=98348 max_rank=0 min=21870 min_rank=4 imbalance_ratio=1.500671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] avg=65536.0000 max=66194 max_rank=5 min=62831 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=43763 max_load_reduction_pct=0.398001 moved_to_replicas=217838 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[98348, 98121, 44536, 32781, 21870, 33412, 96899, 98321] avg=65536.0000 max=98348 max_rank=0 min=21870 min_rank=4 imbalance_ratio=1.500671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] avg=65536.0000 max=66194 max_rank=5 min=62831 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=43763 max_load_reduction_pct=0.398001 moved_to_replicas=217838 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] avg=65536.0000 max=66194 max_rank=5 min=62831 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=43763 max_load_reduction_pct=0.398001 moved_to_replicas=217838 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] avg=65536.0000 max=66194 max_rank=5 min=62831 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=43763 max_load_reduction_pct=0.398001 moved_to_replicas=217838 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32904, 18498, 98253, 94014, 65526, 109957, 32909, 72227] avg=65536.0000 max=109957 max_rank=5 min=18498 min_rank=1 imbalance_ratio=1.677811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] avg=65536.0000 max=66194 max_rank=2 min=63075 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32154 max_load_reduction_pct=0.326941 moved_to_replicas=253732 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[98348, 98121, 44536, 32781, 21870, 33412, 96899, 98321] avg=65536.0000 max=98348 max_rank=0 min=21870 min_rank=4 imbalance_ratio=1.500671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] avg=65536.0000 max=66194 max_rank=2 min=63075 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32154 max_load_reduction_pct=0.326941 moved_to_replicas=253732 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[98348, 98121, 44536, 32781, 21870, 33412, 96899, 98321] avg=65536.0000 max=98348 max_rank=0 min=21870 min_rank=4 imbalance_ratio=1.500671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[66190, 62831, 66192, 66191, 65526, 66194, 64970, 66194] avg=65536.0000 max=66194 max_rank=5 min=62831 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=43763 max_load_reduction_pct=0.398001 moved_to_replicas=217838 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[98348, 98121, 44536, 32781, 21870, 33412, 96899, 98321] avg=65536.0000 max=98348 max_rank=0 min=21870 min_rank=4 imbalance_ratio=1.500671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[98348, 98121, 44536, 32781, 21870, 33412, 96899, 98321] avg=65536.0000 max=98348 max_rank=0 min=21870 min_rank=4 imbalance_ratio=1.500671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[98348, 98121, 44536, 32781, 21870, 33412, 96899, 98321] avg=65536.0000 max=98348 max_rank=0 min=21870 min_rank=4 imbalance_ratio=1.500671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] avg=65536.0000 max=66194 max_rank=2 min=63075 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32154 max_load_reduction_pct=0.326941 moved_to_replicas=253732 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[98278, 182, 34628, 94542, 102168, 63583, 65578, 65329] avg=65536.0000 max=102168 max_rank=4 min=182 min_rank=1 imbalance_ratio=1.558960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] avg=65536.0000 max=66194 max_rank=2 min=63075 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32154 max_load_reduction_pct=0.326941 moved_to_replicas=253732 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[98278, 182, 34628, 94542, 102168, 63583, 65578, 65329] avg=65536.0000 max=102168 max_rank=4 min=182 min_rank=1 imbalance_ratio=1.558960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] avg=65536.0000 max=66194 max_rank=2 min=63075 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32154 max_load_reduction_pct=0.326941 moved_to_replicas=253732 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] avg=65536.0000 max=66194 max_rank=2 min=63075 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32154 max_load_reduction_pct=0.326941 moved_to_replicas=253732 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] avg=65536.0000 max=66194 max_rank=2 min=63075 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32154 max_load_reduction_pct=0.326941 moved_to_replicas=253732 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[98348, 98121, 44536, 32781, 21870, 33412, 96899, 98321] avg=65536.0000 max=98348 max_rank=0 min=21870 min_rank=4 imbalance_ratio=1.500671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] avg=65536.0000 max=66192 max_rank=0 min=63583 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35976 max_load_reduction_pct=0.352126 moved_to_replicas=191672 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[98278, 182, 34628, 94542, 102168, 63583, 65578, 65329] avg=65536.0000 max=102168 max_rank=4 min=182 min_rank=1 imbalance_ratio=1.558960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] avg=65536.0000 max=66192 max_rank=0 min=63583 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35976 max_load_reduction_pct=0.352126 moved_to_replicas=191672 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[98278, 182, 34628, 94542, 102168, 63583, 65578, 65329] avg=65536.0000 max=102168 max_rank=4 min=182 min_rank=1 imbalance_ratio=1.558960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66192, 66194, 64910, 63075, 65341, 66192, 66192] avg=65536.0000 max=66194 max_rank=2 min=63075 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32154 max_load_reduction_pct=0.326941 moved_to_replicas=253732 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[98278, 182, 34628, 94542, 102168, 63583, 65578, 65329] avg=65536.0000 max=102168 max_rank=4 min=182 min_rank=1 imbalance_ratio=1.558960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[98278, 182, 34628, 94542, 102168, 63583, 65578, 65329] avg=65536.0000 max=102168 max_rank=4 min=182 min_rank=1 imbalance_ratio=1.558960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] avg=65536.0000 max=66192 max_rank=0 min=63583 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35976 max_load_reduction_pct=0.352126 moved_to_replicas=191672 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[98278, 182, 34628, 94542, 102168, 63583, 65578, 65329] avg=65536.0000 max=102168 max_rank=4 min=182 min_rank=1 imbalance_ratio=1.558960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[64725, 33255, 130631, 98358, 11, 131710, 32524, 33074] avg=65536.0000 max=131710 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] avg=65536.0000 max=66192 max_rank=0 min=63583 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35976 max_load_reduction_pct=0.352126 moved_to_replicas=191672 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[64725, 33255, 130631, 98358, 11, 131710, 32524, 33074] avg=65536.0000 max=131710 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] avg=65536.0000 max=66192 max_rank=0 min=63583 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35976 max_load_reduction_pct=0.352126 moved_to_replicas=191672 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] avg=65536.0000 max=66192 max_rank=0 min=63583 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35976 max_load_reduction_pct=0.352126 moved_to_replicas=191672 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] avg=65536.0000 max=66192 max_rank=0 min=63583 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35976 max_load_reduction_pct=0.352126 moved_to_replicas=191672 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] avg=65536.0000 max=66192 max_rank=2 min=64725 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65518 max_load_reduction_pct=0.497441 moved_to_replicas=323584 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[98278, 182, 34628, 94542, 102168, 63583, 65578, 65329] avg=65536.0000 max=102168 max_rank=4 min=182 min_rank=1 imbalance_ratio=1.558960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[64725, 33255, 130631, 98358, 11, 131710, 32524, 33074] avg=65536.0000 max=131710 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] avg=65536.0000 max=66192 max_rank=2 min=64725 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65518 max_load_reduction_pct=0.497441 moved_to_replicas=323584 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[64725, 33255, 130631, 98358, 11, 131710, 32524, 33074] avg=65536.0000 max=131710 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[66192, 65031, 66192, 66191, 66192, 63583, 65578, 65329] avg=65536.0000 max=66192 max_rank=0 min=63583 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35976 max_load_reduction_pct=0.352126 moved_to_replicas=191672 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[64725, 33255, 130631, 98358, 11, 131710, 32524, 33074] avg=65536.0000 max=131710 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[64725, 33255, 130631, 98358, 11, 131710, 32524, 33074] avg=65536.0000 max=131710 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] avg=65536.0000 max=66192 max_rank=2 min=64725 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65518 max_load_reduction_pct=0.497441 moved_to_replicas=323584 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[64725, 33255, 130631, 98358, 11, 131710, 32524, 33074] avg=65536.0000 max=131710 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[240, 32767, 428, 98264, 97930, 163377, 65687, 65595] avg=65536.0000 max=163377 max_rank=5 min=240 min_rank=0 imbalance_ratio=2.492935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] avg=65536.0000 max=66192 max_rank=2 min=64725 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65518 max_load_reduction_pct=0.497441 moved_to_replicas=323584 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[240, 32767, 428, 98264, 97930, 163377, 65687, 65595] avg=65536.0000 max=163377 max_rank=5 min=240 min_rank=0 imbalance_ratio=2.492935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] avg=65536.0000 max=66192 max_rank=2 min=64725 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65518 max_load_reduction_pct=0.497441 moved_to_replicas=323584 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] avg=65536.0000 max=66192 max_rank=2 min=64725 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65518 max_load_reduction_pct=0.497441 moved_to_replicas=323584 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] avg=65536.0000 max=66192 max_rank=2 min=64725 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65518 max_load_reduction_pct=0.497441 moved_to_replicas=323584 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] avg=65536.0000 max=66192 max_rank=3 min=64439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97185 max_load_reduction_pct=0.594851 moved_to_replicas=321217 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[64725, 33255, 130631, 98358, 11, 131710, 32524, 33074] avg=65536.0000 max=131710 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[240, 32767, 428, 98264, 97930, 163377, 65687, 65595] avg=65536.0000 max=163377 max_rank=5 min=240 min_rank=0 imbalance_ratio=2.492935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] avg=65536.0000 max=66192 max_rank=3 min=64439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97185 max_load_reduction_pct=0.594851 moved_to_replicas=321217 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[240, 32767, 428, 98264, 97930, 163377, 65687, 65595] avg=65536.0000 max=163377 max_rank=5 min=240 min_rank=0 imbalance_ratio=2.492935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[64725, 65421, 66192, 66192, 65537, 66192, 65279, 64750] avg=65536.0000 max=66192 max_rank=2 min=64725 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65518 max_load_reduction_pct=0.497441 moved_to_replicas=323584 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[240, 32767, 428, 98264, 97930, 163377, 65687, 65595] avg=65536.0000 max=163377 max_rank=5 min=240 min_rank=0 imbalance_ratio=2.492935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] avg=65536.0000 max=66192 max_rank=3 min=64439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97185 max_load_reduction_pct=0.594851 moved_to_replicas=321217 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[240, 32767, 428, 98264, 97930, 163377, 65687, 65595] avg=65536.0000 max=163377 max_rank=5 min=240 min_rank=0 imbalance_ratio=2.492935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[70689, 65476, 48722, 57888, 115, 85639, 98226, 97533] avg=65536.0000 max=98226 max_rank=6 min=115 min_rank=4 imbalance_ratio=1.498810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[240, 32767, 428, 98264, 97930, 163377, 65687, 65595] avg=65536.0000 max=163377 max_rank=5 min=240 min_rank=0 imbalance_ratio=2.492935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] avg=65536.0000 max=66192 max_rank=3 min=64439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97185 max_load_reduction_pct=0.594851 moved_to_replicas=321217 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[70689, 65476, 48722, 57888, 115, 85639, 98226, 97533] avg=65536.0000 max=98226 max_rank=6 min=115 min_rank=4 imbalance_ratio=1.498810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] avg=65536.0000 max=66192 max_rank=3 min=64439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97185 max_load_reduction_pct=0.594851 moved_to_replicas=321217 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] avg=65536.0000 max=66192 max_rank=3 min=64439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97185 max_load_reduction_pct=0.594851 moved_to_replicas=321217 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[240, 32767, 428, 98264, 97930, 163377, 65687, 65595] avg=65536.0000 max=163377 max_rank=5 min=240 min_rank=0 imbalance_ratio=2.492935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63490 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32034 max_load_reduction_pct=0.326125 moved_to_replicas=176078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] avg=65536.0000 max=66192 max_rank=3 min=64439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97185 max_load_reduction_pct=0.594851 moved_to_replicas=321217 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[70689, 65476, 48722, 57888, 115, 85639, 98226, 97533] avg=65536.0000 max=98226 max_rank=6 min=115 min_rank=4 imbalance_ratio=1.498810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63490 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32034 max_load_reduction_pct=0.326125 moved_to_replicas=176078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[70689, 65476, 48722, 57888, 115, 85639, 98226, 97533] avg=65536.0000 max=98226 max_rank=6 min=115 min_rank=4 imbalance_ratio=1.498810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[65076, 64439, 64915, 66192, 66192, 66192, 65687, 65595] avg=65536.0000 max=66192 max_rank=3 min=64439 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97185 max_load_reduction_pct=0.594851 moved_to_replicas=321217 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[70689, 65476, 48722, 57888, 115, 85639, 98226, 97533] avg=65536.0000 max=98226 max_rank=6 min=115 min_rank=4 imbalance_ratio=1.498810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63490 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32034 max_load_reduction_pct=0.326125 moved_to_replicas=176078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[70689, 65476, 48722, 57888, 115, 85639, 98226, 97533] avg=65536.0000 max=98226 max_rank=6 min=115 min_rank=4 imbalance_ratio=1.498810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32749, 32769, 130981, 32809, 98536, 98148, 65520, 32776] avg=65536.0000 max=130981 max_rank=2 min=32749 min_rank=0 imbalance_ratio=1.998611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[70689, 65476, 48722, 57888, 115, 85639, 98226, 97533] avg=65536.0000 max=98226 max_rank=6 min=115 min_rank=4 imbalance_ratio=1.498810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63490 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32034 max_load_reduction_pct=0.326125 moved_to_replicas=176078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32749, 32769, 130981, 32809, 98536, 98148, 65520, 32776] avg=65536.0000 max=130981 max_rank=2 min=32749 min_rank=0 imbalance_ratio=1.998611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63490 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32034 max_load_reduction_pct=0.326125 moved_to_replicas=176078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63490 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32034 max_load_reduction_pct=0.326125 moved_to_replicas=176078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] avg=65536.0000 max=66192 max_rank=2 min=64765 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64789 max_load_reduction_pct=0.494644 moved_to_replicas=257992 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[70689, 65476, 48722, 57888, 115, 85639, 98226, 97533] avg=65536.0000 max=98226 max_rank=6 min=115 min_rank=4 imbalance_ratio=1.498810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32749, 32769, 130981, 32809, 98536, 98148, 65520, 32776] avg=65536.0000 max=130981 max_rank=2 min=32749 min_rank=0 imbalance_ratio=1.998611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63490 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32034 max_load_reduction_pct=0.326125 moved_to_replicas=176078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] avg=65536.0000 max=66192 max_rank=2 min=64765 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64789 max_load_reduction_pct=0.494644 moved_to_replicas=257992 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32749, 32769, 130981, 32809, 98536, 98148, 65520, 32776] avg=65536.0000 max=130981 max_rank=2 min=32749 min_rank=0 imbalance_ratio=1.998611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[66191, 65476, 66192, 64363, 63490, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63490 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32034 max_load_reduction_pct=0.326125 moved_to_replicas=176078 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] avg=65536.0000 max=66192 max_rank=2 min=64765 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64789 max_load_reduction_pct=0.494644 moved_to_replicas=257992 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32749, 32769, 130981, 32809, 98536, 98148, 65520, 32776] avg=65536.0000 max=130981 max_rank=2 min=32749 min_rank=0 imbalance_ratio=1.998611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32749, 32769, 130981, 32809, 98536, 98148, 65520, 32776] avg=65536.0000 max=130981 max_rank=2 min=32749 min_rank=0 imbalance_ratio=1.998611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[88540, 35335, 65874, 98246, 32779, 32963, 65524, 105027] avg=65536.0000 max=105027 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.602585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32749, 32769, 130981, 32809, 98536, 98148, 65520, 32776] avg=65536.0000 max=130981 max_rank=2 min=32749 min_rank=0 imbalance_ratio=1.998611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] avg=65536.0000 max=66192 max_rank=2 min=64765 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64789 max_load_reduction_pct=0.494644 moved_to_replicas=257992 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[88540, 35335, 65874, 98246, 32779, 32963, 65524, 105027] avg=65536.0000 max=105027 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.602585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] avg=65536.0000 max=66192 max_rank=2 min=64765 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64789 max_load_reduction_pct=0.494644 moved_to_replicas=257992 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] avg=65536.0000 max=66192 max_rank=2 min=64765 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64789 max_load_reduction_pct=0.494644 moved_to_replicas=257992 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] avg=65536.0000 max=66194 max_rank=7 min=62582 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=38833 max_load_reduction_pct=0.369743 moved_to_replicas=183640 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32749, 32769, 130981, 32809, 98536, 98148, 65520, 32776] avg=65536.0000 max=130981 max_rank=2 min=32749 min_rank=0 imbalance_ratio=1.998611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[88540, 35335, 65874, 98246, 32779, 32963, 65524, 105027] avg=65536.0000 max=105027 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.602585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] avg=65536.0000 max=66192 max_rank=2 min=64765 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64789 max_load_reduction_pct=0.494644 moved_to_replicas=257992 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] avg=65536.0000 max=66194 max_rank=7 min=62582 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=38833 max_load_reduction_pct=0.369743 moved_to_replicas=183640 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[65345, 64962, 66192, 64765, 66192, 66192, 65520, 65120] avg=65536.0000 max=66192 max_rank=2 min=64765 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64789 max_load_reduction_pct=0.494644 moved_to_replicas=257992 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[88540, 35335, 65874, 98246, 32779, 32963, 65524, 105027] avg=65536.0000 max=105027 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.602585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] avg=65536.0000 max=66194 max_rank=7 min=62582 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=38833 max_load_reduction_pct=0.369743 moved_to_replicas=183640 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[88540, 35335, 65874, 98246, 32779, 32963, 65524, 105027] avg=65536.0000 max=105027 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.602585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[88540, 35335, 65874, 98246, 32779, 32963, 65524, 105027] avg=65536.0000 max=105027 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.602585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[88540, 35335, 65874, 98246, 32779, 32963, 65524, 105027] avg=65536.0000 max=105027 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.602585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] avg=65536.0000 max=66194 max_rank=7 min=62582 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=38833 max_load_reduction_pct=0.369743 moved_to_replicas=183640 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] avg=65536.0000 max=66194 max_rank=7 min=62582 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=38833 max_load_reduction_pct=0.369743 moved_to_replicas=183640 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] avg=65536.0000 max=66194 max_rank=7 min=62582 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=38833 max_load_reduction_pct=0.369743 moved_to_replicas=183640 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[88540, 35335, 65874, 98246, 32779, 32963, 65524, 105027] avg=65536.0000 max=105027 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.602585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] avg=65536.0000 max=66194 max_rank=7 min=62582 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=38833 max_load_reduction_pct=0.369743 moved_to_replicas=183640 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[66192, 66192, 65874, 66192, 65538, 62582, 65524, 66194] avg=65536.0000 max=66194 max_rank=7 min=62582 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=38833 max_load_reduction_pct=0.369743 moved_to_replicas=183640 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=2102614 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=103936 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=2097237 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=109312 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=2082261 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=125440 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=2095320 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=113536 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=2098906 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=108160 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=2099521 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=108544 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=2096820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=111104 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=2104537 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=102272 peak_demand=66194 required_capacity=1.010040 + [2026-08-30 05:15:28] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 527.9 | throughput per GPU (TFLOP/s/GPU): 631.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.875321E+00 | loss scale: 1.0 | grad norm: 5.308 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +[after training is done] datetime: 2026-08-30 05:15:28 +[rank0]:[W830 05:15:30.570028703 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 05:15:31.098493110 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 05:15:31.136435004 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 05:15:31.141055429 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 05:15:31.142575350 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 05:15:31.144659352 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 05:15:31.145092140 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 05:15:32.709120071 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:15:34.492000 498940 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:15:34.493000 498940 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 499032 closing signal SIGTERM +W0830 05:15:34.493000 498940 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 499033 closing signal SIGTERM +W0830 05:15:34.494000 498940 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 499034 closing signal SIGTERM +W0830 05:15:34.494000 498940 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 499035 closing signal SIGTERM +W0830 05:15:34.494000 498940 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 499036 closing signal SIGTERM +W0830 05:15:34.495000 498940 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 499037 closing signal SIGTERM +W0830 05:15:34.495000 498940 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 499038 closing signal SIGTERM +W0830 05:15:34.496000 498940 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 499039 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 498940 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..1dad6e2a4c1d80628f71364600d9274a2c92c157 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,224 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 4 +1, 4 +2, 165 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 2301 +1, 2307 +2, 2307 +3, 2301 +4, 2315 +5, 2309 +6, 2313 +7, 2323 +0, 35145 +1, 35145 +2, 35145 +3, 35145 +4, 35145 +5, 35145 +6, 35145 +7, 35145 +0, 42227 +1, 42213 +2, 42213 +3, 42213 +4, 42213 +5, 42213 +6, 42213 +7, 42213 +0, 43545 +1, 43529 +2, 43529 +3, 43529 +4, 43529 +5, 43529 +6, 43529 +7, 43529 +0, 43545 +1, 43529 +2, 43529 +3, 43529 +4, 43529 +5, 43529 +6, 43529 +7, 43529 +0, 44185 +1, 44169 +2, 44169 +3, 44169 +4, 44169 +5, 44169 +6, 44169 +7, 44169 +0, 46451 +1, 46435 +2, 46435 +3, 46435 +4, 46435 +5, 46435 +6, 46435 +7, 46435 +0, 54689 +1, 54671 +2, 54643 +3, 54673 +4, 54757 +5, 54673 +6, 54701 +7, 54687 +0, 76579 +1, 76563 +2, 76563 +3, 76563 +4, 76633 +5, 76563 +6, 76563 +7, 76563 +0, 76579 +1, 76563 +2, 76563 +3, 76563 +4, 76633 +5, 76563 +6, 76563 +7, 76563 +0, 76579 +1, 76563 +2, 76563 +3, 76563 +4, 76633 +5, 76563 +6, 76563 +7, 76563 +0, 89049 +1, 89033 +2, 89033 +3, 89033 +4, 89083 +5, 89033 +6, 89033 +7, 89033 +0, 90195 +1, 90201 +2, 90219 +3, 90229 +4, 90267 +5, 90191 +6, 90229 +7, 90197 +0, 91359 +1, 91327 +2, 91325 +3, 91321 +4, 91417 +5, 91347 +6, 91331 +7, 91357 +0, 114205 +1, 114249 +2, 114249 +3, 114249 +4, 114319 +5, 114249 +6, 114249 +7, 114249 +0, 114205 +1, 114249 +2, 114249 +3, 114249 +4, 114319 +5, 114249 +6, 114249 +7, 114249 +0, 114205 +1, 114249 +2, 114249 +3, 114249 +4, 114319 +5, 114249 +6, 114249 +7, 114249 +0, 110711 +1, 110755 +2, 110755 +3, 110755 +4, 110825 +5, 110755 +6, 110755 +7, 110755 +0, 109031 +1, 109075 +2, 109075 +3, 109075 +4, 109145 +5, 108751 +6, 109721 +7, 109395 +0, 4 +1, 107395 +2, 107459 +3, 108041 +4, 324 +5, 4 +6, 4 +7, 4 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..1dd58510aa5526407243c143c4c1579c046da902 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G3 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=1792 +num_experts=64 +topk=16 +experts_per_rank=8 +expert_params=22020096 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=65536 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2af33ef9aef43e692514496dff554663a0f5b4b8 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log @@ -0,0 +1,9092 @@ +[05:13:09] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G3 / ultraep (slots/rank=8 policy=matched) + E=64 K=16 H'=1792 m=8 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:13:11.216000 495393 torch/distributed/run.py:766] +W0830 05:13:11.216000 495393 torch/distributed/run.py:766] ***************************************** +W0830 05:13:11.216000 495393 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:13:11.216000 495393 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1792 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 16 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 45 is connected to is connected to 77 peer ranks. [Gloo] Rank peer ranks. Expected number of connected peer ranks is : 6Expected number of connected peer ranks is : 7 is connected to 77 peer ranks. +Expected number of connected peer ranks is : 7 + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 631[Gloo] Rank 245 is connected to is connected to is connected to is connected to 7 is connected to is connected to 7777 is connected to 77 peer ranks. peer ranks. peer ranks. peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7777Expected number of connected peer ranks is : 77 + + + +7 + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank [Gloo] Rank 00 is connected to [Gloo] Rank is connected to 000 is connected to peer ranks. peer ranks. 0Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. 00 +Expected number of connected peer ranks is : +0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W830 05:13:29.552659336 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:13:29.552973858 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:13:29.555317373 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:13:29.555351766 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:13:29.555398451 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:13:29.555518899 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:13:29.555426234 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:13:29.707810427 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.024 seconds +time to initialize megatron (seconds): 7.278 +[after megatron is initialized] datetime: 2026-08-30 05:13:35 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6724653056 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42217472 elements, 42217472 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (42213376 elements, 42213376 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight +Params for bucket 4 (42213376 elements, 42213376 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight +Params for bucket 5 (42213376 elements, 42213376 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 6 (42213376 elements, 42213376 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight +Params for bucket 7 (42213376 elements, 42213376 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 8 (42213376 elements, 42213376 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight +Params for bucket 9 (42213376 elements, 42213376 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 10 (42213376 elements, 42213376 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 11 (42213376 elements, 42213376 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 12 (42213376 elements, 42213376 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 13 (42213376 elements, 42213376 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (42213376 elements, 42213376 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 15 (42213376 elements, 42213376 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight +Params for bucket 16 (42213376 elements, 42213376 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (42213376 elements, 42213376 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 64 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 17 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 31 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 35 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 42 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 55 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 +Params for bucket 56 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 63 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:13:38 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:13:39 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3490.48, 3496.51) + train/valid/test-data-iterators-setup ..........: (72.82, 90.81) +[before the start of training step] datetime: 2026-08-30 05:13:39 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131115 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131115 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=81920 expected_recv=126378 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131190 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=81920 expected_recv=126378 actual_valid_recv=127360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=81920 expected_recv=130356 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131190 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=127360 slack_slots=36480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=81920 expected_recv=130356 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=81920 expected_recv=129879 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=81920 expected_recv=129879 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=81920 expected_recv=129688 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=81920 expected_recv=129688 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=81920 expected_recv=127091 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=81920 expected_recv=127091 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131069 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=128384 slack_slots=35456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=81920 expected_recv=128043 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131069 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=81920 expected_recv=128043 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=81920 expected_recv=125849 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=129280 slack_slots=34560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=81920 expected_recv=125849 actual_valid_recv=126976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=126976 slack_slots=36864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131332 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131332 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=81920 expected_recv=128476 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=81920 expected_recv=128476 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=81920 expected_recv=130137 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=129408 slack_slots=34432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=81920 expected_recv=130137 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=81920 expected_recv=130737 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=81920 expected_recv=130165 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131844 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=81920 expected_recv=129984 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=81920 expected_recv=130737 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=81920 expected_recv=130165 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131844 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=81920 expected_recv=129984 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=81920 expected_recv=127113 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=81920 expected_recv=129831 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=81920 expected_recv=127113 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=81920 expected_recv=129831 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=128384 slack_slots=35456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=81920 expected_recv=126976 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=81920 expected_recv=126976 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=81920 expected_recv=128694 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=128128 slack_slots=35712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=81920 expected_recv=128694 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=81920 expected_recv=130554 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=81920 expected_recv=130554 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=81920 expected_recv=129848 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=81920 expected_recv=129418 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=81920 expected_recv=128101 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=81920 expected_recv=129848 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=81920 expected_recv=129418 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=81920 expected_recv=128101 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=129408 slack_slots=34432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=81920 expected_recv=128989 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=81920 expected_recv=128989 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=81920 expected_recv=129776 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=81920 expected_recv=129776 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=81920 expected_recv=127463 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=81920 expected_recv=127463 actual_valid_recv=128896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=81920 expected_recv=130791 expected_overflow=0 exact_recv_per_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=128896 slack_slots=34944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=81920 expected_recv=128348 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=81920 expected_recv=130791 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=81920 expected_recv=128348 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=129664 slack_slots=34176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=81920 expected_recv=126494 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=81920 expected_recv=127777 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=81920 expected_recv=126494 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=81920 expected_recv=127777 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=128128 slack_slots=35712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=128768 slack_slots=35072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131602 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=81920 expected_recv=129600 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131602 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=81920 expected_recv=129600 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131018 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131018 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=81920 expected_recv=132095 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=81920 expected_recv=132095 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=81920 expected_recv=128342 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=81920 expected_recv=128456 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131246 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=81920 expected_recv=128342 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=81920 expected_recv=128456 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=81920 expected_recv=128812 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131246 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=129792 slack_slots=34048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=81920 expected_recv=128812 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131312 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131312 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=81920 expected_recv=128261 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131519 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=81920 expected_recv=128261 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131519 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=81920 expected_recv=129890 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=129408 slack_slots=34432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=81920 expected_recv=128426 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=81920 expected_recv=129890 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=81920 expected_recv=128426 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=129792 slack_slots=34048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=81920 expected_recv=128910 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131262 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=81920 expected_recv=128910 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131262 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=81920 expected_recv=130066 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=81920 expected_recv=130577 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=81920 expected_recv=130066 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=81920 expected_recv=130577 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=81920 expected_recv=130270 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=81920 expected_recv=130606 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=81920 expected_recv=130270 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=81920 expected_recv=130208 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=81920 expected_recv=130606 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=81920 expected_recv=130208 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=81920 expected_recv=128409 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=81920 expected_recv=128409 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=81920 expected_recv=127848 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131260 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=81920 expected_recv=127848 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=81920 expected_recv=130509 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131192 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=129280 slack_slots=34560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131260 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=81920 expected_recv=130509 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131192 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=81920 expected_recv=127192 expected_overflow=0 exact_recv_per_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=81920 expected_recv=127192 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=128512 slack_slots=35328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131111 expected_overflow=0 exact_recv_per_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131111 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131605 expected_overflow=0 exact_recv_per_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131605 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[61778, 46459, 71006, 70251, 79422, 85882, 42571, 66919] avg=65536.0000 max=85882 max_rank=5 min=42571 min_rank=6 imbalance_ratio=1.310455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[65837, 64502, 66192, 66192, 66193, 66192, 62988, 66192] avg=65536.0000 max=66193 max_rank=4 min=62988 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=19689 max_load_reduction_pct=0.229256 moved_to_replicas=84431 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[97320, 24454, 74250, 86598, 85524, 32020, 70175, 53947] avg=65536.0000 max=97320 max_rank=0 min=24454 min_rank=1 imbalance_ratio=1.484985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[66192, 63640, 66192, 66192, 66192, 66192, 66193, 63495] avg=65536.0000 max=66193 max_rank=6 min=63495 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=31127 max_load_reduction_pct=0.319842 moved_to_replicas=158215 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[114176, 36698, 52573, 54593, 60999, 93862, 33731, 77656] avg=65536.0000 max=114176 max_rank=0 min=33731 min_rank=6 imbalance_ratio=1.742188 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[66191, 65140, 66192, 66192, 63312, 66192, 64877, 66192] avg=65536.0000 max=66192 max_rank=2 min=63312 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=47984 max_load_reduction_pct=0.420263 moved_to_replicas=179816 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[57987, 53980, 51038, 40741, 55906, 116763, 74577, 73296] avg=65536.0000 max=116763 max_rank=5 min=40741 min_rank=3 imbalance_ratio=1.781662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[65091, 63946, 66192, 66192, 64292, 66192, 66191, 66192] avg=65536.0000 max=66192 max_rank=2 min=63946 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50571 max_load_reduction_pct=0.433108 moved_to_replicas=120122 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[99853, 70321, 38134, 88787, 52043, 54691, 46455, 74004] avg=65536.0000 max=99853 max_rank=0 min=38134 min_rank=2 imbalance_ratio=1.523636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 62502, 64633, 66193] avg=65536.0000 max=66193 max_rank=7 min=62502 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=33660 max_load_reduction_pct=0.337096 moved_to_replicas=126695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[41473, 63480, 130426, 95242, 28698, 35016, 88661, 41292] avg=65536.0000 max=130426 max_rank=2 min=28698 min_rank=4 imbalance_ratio=1.990143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[66192, 63480, 66192, 66192, 66192, 63656, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63480 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64234 max_load_reduction_pct=0.492494 moved_to_replicas=213514 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32271, 117185, 75198, 47445, 64362, 52035, 77627, 58165] avg=65536.0000 max=117185 max_rank=1 min=32271 min_rank=0 imbalance_ratio=1.788101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[64106, 66192, 66192, 66192, 64362, 64860, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=64106 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=50993 max_load_reduction_pct=0.435150 moved_to_replicas=145432 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[92622, 69907, 93388, 26809, 73788, 31142, 64908, 71724] avg=65536.0000 max=93388 max_rank=2 min=26809 min_rank=3 imbalance_ratio=1.424988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[66192, 66193, 66192, 65316, 66191, 63104, 64908, 66192] avg=65536.0000 max=66193 max_rank=1 min=63104 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=27195 max_load_reduction_pct=0.291204 moved_to_replicas=103907 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[60262, 46258, 68240, 74214, 61537, 94412, 75982, 43383] avg=65536.0000 max=94412 max_rank=5 min=43383 min_rank=7 imbalance_ratio=1.440613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[66193, 63549, 66192, 66191, 63585, 66193, 66193, 66192] avg=65536.0000 max=66193 max_rank=0 min=63549 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=28219 max_load_reduction_pct=0.298892 moved_to_replicas=102867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[19997, 81205, 34039, 69094, 99910, 50742, 84437, 84864] avg=65536.0000 max=99910 max_rank=4 min=19997 min_rank=0 imbalance_ratio=1.524506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[66193, 66191, 60945, 66192, 66192, 66191, 66193, 66191] avg=65536.0000 max=66193 max_rank=0 min=60945 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=33717 max_load_reduction_pct=0.337474 moved_to_replicas=170024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[99203, 58495, 70633, 55123, 64826, 68060, 43271, 64677] avg=65536.0000 max=99203 max_rank=0 min=43271 min_rank=6 imbalance_ratio=1.513718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[66192, 65902, 66192, 64114, 64826, 66192, 66193, 64677] avg=65536.0000 max=66193 max_rank=6 min=64114 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33010 max_load_reduction_pct=0.332752 moved_to_replicas=84403 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=2089759 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=511872 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[62784, 65121, 75315, 100092, 49944, 82009, 75943, 13080] avg=65536.0000 max=100092 max_rank=3 min=13080 min_rank=7 imbalance_ratio=1.527283 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[65410, 65121, 66191, 66191, 66193, 66190, 66192, 62800] avg=65536.0000 max=66193 max_rank=4 min=62800 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=33899 max_load_reduction_pct=0.338678 moved_to_replicas=120970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=2101251 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=500992 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=2093418 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=511360 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=2095336 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506240 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[59194, 82940, 61945, 90431, 52400, 35404, 81086, 60888] avg=65536.0000 max=90431 max_rank=3 min=35404 min_rank=5 imbalance_ratio=1.379868 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=2085444 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=516096 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[62150, 66191, 64987, 66193, 66193, 66190, 66191, 66193] avg=65536.0000 max=66193 max_rank=3 min=62150 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=24238 max_load_reduction_pct=0.268028 moved_to_replicas=100450 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=2102385 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=501248 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[61399, 43371, 63873, 68129, 84256, 46838, 93850, 62572] avg=65536.0000 max=93850 max_rank=6 min=43371 min_rank=1 imbalance_ratio=1.432037 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[66193, 64385, 63873, 66192, 66191, 66193, 66191, 65070] avg=65536.0000 max=66193 max_rank=0 min=63873 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=27657 max_load_reduction_pct=0.294694 moved_to_replicas=103703 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[56946, 36583, 72633, 64918, 80125, 141318, 21884, 49881] avg=65536.0000 max=141318 max_rank=5 min=21884 min_rank=6 imbalance_ratio=2.156342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[66190, 62218, 66193, 64918, 66193, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=62218 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=75125 max_load_reduction_pct=0.531602 moved_to_replicas=182637 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[59381, 86940, 65384, 28894, 72406, 87383, 81331, 42569] avg=65536.0000 max=87383 max_rank=5 min=28894 min_rank=3 imbalance_ratio=1.333359 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=2102492 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=499328 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[65595, 66192, 65384, 65413, 66192, 66192, 66192, 63128] avg=65536.0000 max=66192 max_rank=1 min=63128 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=21191 max_load_reduction_pct=0.242507 moved_to_replicas=104055 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=2107131 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=495616 peak_demand=66193 required_capacity=1.010025 +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 [2026-08-30 05:14:08] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 29038.3 | throughput per GPU (TFLOP/s/GPU): 11.5 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102352E+01 | loss scale: 1.0 | grad norm: 44.192 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67072 | min safe capacity: 1.0234 | + +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58092.3076171875 | max allocated: 79553.50390625 | reserved: 82188.0 | max reserved: 82188.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=81920 expected_recv=128740 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=81920 expected_recv=129100 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=81920 expected_recv=129985 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131215 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=81920 expected_recv=128740 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=81920 expected_recv=129100 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=81920 expected_recv=129985 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=130048 slack_slots=33792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131215 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=81920 expected_recv=127240 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=81920 expected_recv=127719 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131700 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=81920 expected_recv=127240 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=81920 expected_recv=127719 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=128768 slack_slots=35072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131700 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=129152 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=81920 expected_recv=127528 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=81920 expected_recv=126744 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=81920 expected_recv=127528 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=81920 expected_recv=126744 actual_valid_recv=128000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=129024 slack_slots=34816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=128000 slack_slots=35840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131353 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=81920 expected_recv=128168 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131353 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=81920 expected_recv=128168 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=129664 slack_slots=34176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131287 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=81920 expected_recv=128228 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131287 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=81920 expected_recv=128228 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=81920 expected_recv=124536 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=129664 slack_slots=34176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=81920 expected_recv=129757 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=81920 expected_recv=124536 actual_valid_recv=125696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=81920 expected_recv=129757 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=125696 slack_slots=38144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132363 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132363 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130094 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130094 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130813 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130421 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130813 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=81920 expected_recv=127711 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130421 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=81920 expected_recv=127711 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=81920 expected_recv=129496 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=129408 slack_slots=34432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=81920 expected_recv=129496 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=81920 expected_recv=124774 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=81920 expected_recv=124774 actual_valid_recv=126208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=126208 slack_slots=37632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=81920 expected_recv=129259 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=81920 expected_recv=129259 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=81920 expected_recv=130261 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=81920 expected_recv=130844 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=81920 expected_recv=130261 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=81920 expected_recv=130844 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=81920 expected_recv=130613 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=81920 expected_recv=128679 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=81920 expected_recv=129804 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131488 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=81920 expected_recv=130613 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=81920 expected_recv=128679 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=81920 expected_recv=129804 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131488 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=81920 expected_recv=129448 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=81920 expected_recv=129448 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=81920 expected_recv=130756 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=81920 expected_recv=130756 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=81920 expected_recv=130072 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=81920 expected_recv=127293 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131350 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=81920 expected_recv=130072 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=81920 expected_recv=127293 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=81920 expected_recv=129640 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131350 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=128512 slack_slots=35328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=81920 expected_recv=129640 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=81920 expected_recv=131266 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=81920 expected_recv=126770 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=81920 expected_recv=131266 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=81920 expected_recv=126770 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=81920 expected_recv=128757 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=128128 slack_slots=35712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=81920 expected_recv=128757 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=81920 expected_recv=125982 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=81920 expected_recv=125982 actual_valid_recv=127360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=127360 slack_slots=36480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=81920 expected_recv=130759 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=134272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=81920 expected_recv=132248 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=81920 expected_recv=130759 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=134272 slack_slots=29568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=81920 expected_recv=129913 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=81920 expected_recv=132248 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=81920 expected_recv=129913 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=81920 expected_recv=131780 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=81920 expected_recv=131780 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=81920 expected_recv=129341 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=81920 expected_recv=129341 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=81920 expected_recv=129838 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130900 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=81920 expected_recv=128080 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=81920 expected_recv=129838 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130900 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=81920 expected_recv=128080 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=129152 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=81920 expected_recv=132065 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=81920 expected_recv=132065 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130247 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=81920 expected_recv=129272 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=81920 expected_recv=129740 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130247 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=81920 expected_recv=129272 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=81920 expected_recv=129740 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=81920 expected_recv=127628 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=81920 expected_recv=127628 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=129152 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=81920 expected_recv=132363 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=81920 expected_recv=129307 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=81920 expected_recv=132363 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=81920 expected_recv=128938 expected_overflow=0 exact_recv_per_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=81920 expected_recv=132317 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=81920 expected_recv=129307 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=81920 expected_recv=128938 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=81920 expected_recv=132317 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=81920 expected_recv=130024 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131091 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=81920 expected_recv=130024 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131091 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=81920 expected_recv=125607 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=81920 expected_recv=125607 actual_valid_recv=126976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=126976 slack_slots=36864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[60170, 44643, 71024, 74589, 78093, 88006, 43970, 63793] avg=65536.0000 max=88006 max_rank=5 min=43970 min_rank=6 imbalance_ratio=1.342865 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[65001, 65223, 66193, 66191, 66192, 66192, 65503, 63793] avg=65536.0000 max=66193 max_rank=2 min=63793 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=21813 max_load_reduction_pct=0.247858 moved_to_replicas=90981 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[89930, 26964, 74230, 84039, 86844, 40873, 72045, 49363] avg=65536.0000 max=89930 max_rank=0 min=26964 min_rank=1 imbalance_ratio=1.372223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[66192, 65610, 66192, 66191, 66192, 61525, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=61525 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=23737 max_load_reduction_pct=0.263950 moved_to_replicas=156287 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[114339, 41715, 41731, 67399, 59428, 93859, 37521, 68296] avg=65536.0000 max=114339 max_rank=0 min=37521 min_rank=6 imbalance_ratio=1.744675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[66192, 63925, 66192, 66192, 63211, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=48147 max_load_reduction_pct=0.421090 moved_to_replicas=165510 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[47348, 51393, 44256, 49698, 60080, 108534, 77762, 85217] avg=65536.0000 max=108534 max_rank=5 min=44256 min_rank=2 imbalance_ratio=1.656097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[66192, 66192, 66191, 62038, 65095, 66194, 66193, 66193] avg=65536.0000 max=66194 max_rank=5 min=62038 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=42340 max_load_reduction_pct=0.390108 moved_to_replicas=131701 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[81707, 76940, 37922, 82321, 50937, 71998, 54687, 67776] avg=65536.0000 max=82321 max_rank=3 min=37922 min_rank=2 imbalance_ratio=1.256119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[66191, 66192, 65058, 66193, 66193, 66192, 62077, 66192] avg=65536.0000 max=66193 max_rank=3 min=62077 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=16128 max_load_reduction_pct=0.195916 moved_to_replicas=96776 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[55822, 57461, 119982, 73711, 31823, 64620, 85359, 35510] avg=65536.0000 max=119982 max_rank=2 min=31823 min_rank=4 imbalance_ratio=1.830780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[66192, 66191, 66192, 66193, 62516, 64620, 66193, 66191] avg=65536.0000 max=66193 max_rank=3 min=62516 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=53789 max_load_reduction_pct=0.448309 moved_to_replicas=170450 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[32311, 131006, 89528, 40182, 53977, 46008, 83694, 47582] avg=65536.0000 max=131006 max_rank=1 min=32311 min_rank=0 imbalance_ratio=1.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[63304, 66193, 66192, 66192, 66193, 63831, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=63304 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64813 max_load_reduction_pct=0.494733 moved_to_replicas=182123 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[82923, 82842, 89197, 41380, 60822, 41065, 44314, 81745] avg=65536.0000 max=89197 max_rank=2 min=41065 min_rank=5 imbalance_ratio=1.361038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[66191, 66192, 66193, 66194, 66191, 64069, 63066, 66192] avg=65536.0000 max=66194 max_rank=3 min=63066 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=23003 max_load_reduction_pct=0.257890 moved_to_replicas=146223 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[58161, 69666, 65295, 71039, 74897, 88771, 59464, 36995] avg=65536.0000 max=88771 max_rank=5 min=36995 min_rank=7 imbalance_ratio=1.354538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[66192, 66192, 65295, 66192, 66192, 66192, 63612, 64421] avg=65536.0000 max=66192 max_rank=0 min=63612 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=22579 max_load_reduction_pct=0.254351 moved_to_replicas=102436 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[17596, 63448, 51441, 66875, 107397, 49810, 67254, 100467] avg=65536.0000 max=107397 max_rank=4 min=17596 min_rank=0 imbalance_ratio=1.638748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[66075, 63448, 63803, 66192, 66193, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=4 min=63448 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=41204 max_load_reduction_pct=0.383661 moved_to_replicas=142415 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[87735, 48647, 69275, 66056, 75196, 73742, 53195, 50442] avg=65536.0000 max=87735 max_rank=0 min=48647 min_rank=1 imbalance_ratio=1.338730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[66192, 66191, 66192, 66056, 66192, 66192, 62199, 65074] avg=65536.0000 max=66192 max_rank=0 min=62199 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=21543 max_load_reduction_pct=0.245546 moved_to_replicas=95494 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[53445, 78611, 59448, 74586, 75315, 76883, 71115, 34885] avg=65536.0000 max=78611 max_rank=1 min=34885 min_rank=7 imbalance_ratio=1.199509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[62568, 66191, 64567, 66191, 66192, 66192, 66193, 66194] avg=65536.0000 max=66194 max_rank=7 min=62568 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=12417 max_load_reduction_pct=0.157955 moved_to_replicas=106751 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[78572, 77035, 63805, 76070, 47007, 39489, 73524, 68786] avg=65536.0000 max=78572 max_rank=0 min=39489 min_rank=5 imbalance_ratio=1.198914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[66191, 66192, 63805, 66192, 65181, 64342, 66193, 66192] avg=65536.0000 max=66193 max_rank=6 min=63805 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=12379 max_load_reduction_pct=0.157550 moved_to_replicas=75033 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[60537, 54745, 68319, 62745, 74316, 59911, 77841, 65874] avg=65536.0000 max=77841 max_rank=6 min=54745 min_rank=1 imbalance_ratio=1.187759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[64709, 66192, 66192, 62745, 66192, 66192, 66192, 65874] avg=65536.0000 max=66192 max_rank=1 min=62745 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11649 max_load_reduction_pct=0.149651 moved_to_replicas=86073 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[50805, 52839, 62855, 74111, 93123, 146668, 17076, 26811] avg=65536.0000 max=146668 max_rank=5 min=17076 min_rank=6 imbalance_ratio=2.237976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[66192, 66192, 63115, 66192, 66192, 66192, 66192, 64021] avg=65536.0000 max=66192 max_rank=0 min=63115 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80476 max_load_reduction_pct=0.548695 moved_to_replicas=223288 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[58343, 71332, 92755, 32182, 63305, 84170, 61828, 60373] avg=65536.0000 max=92755 max_rank=2 min=32182 min_rank=3 imbalance_ratio=1.415329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 63305, 66192, 66191, 63831] avg=65536.0000 max=66193 max_rank=1 min=63305 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=26562 max_load_reduction_pct=0.286367 moved_to_replicas=93224 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=2098354 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=503424 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=2094402 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508032 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=2097384 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=504704 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=2093999 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508416 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=2095632 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507008 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=2097571 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=504832 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=2095485 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506624 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=2104389 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=499712 peak_demand=66194 required_capacity=1.010040 + [2026-08-30 05:14:08] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 753.3 | throughput per GPU (TFLOP/s/GPU): 442.6 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101566E+01 | loss scale: 1.0 | grad norm: 45.944 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=81920 expected_recv=127722 expected_overflow=0 exact_recv_per_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=81920 expected_recv=126550 expected_overflow=0 exact_recv_per_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=81920 expected_recv=127722 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=81920 expected_recv=126550 actual_valid_recv=127872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=129152 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=127872 slack_slots=35968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=81920 expected_recv=131176 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=81920 expected_recv=127049 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=81920 expected_recv=131176 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=81920 expected_recv=128432 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=81920 expected_recv=127049 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=128384 slack_slots=35456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=81920 expected_recv=128432 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=81920 expected_recv=127768 expected_overflow=0 exact_recv_per_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=81920 expected_recv=127768 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=81920 expected_recv=129018 expected_overflow=0 exact_recv_per_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=129280 slack_slots=34560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=81920 expected_recv=129018 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=81920 expected_recv=129871 expected_overflow=0 exact_recv_per_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=81920 expected_recv=129871 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130037 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130037 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130462 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=81920 expected_recv=127135 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130462 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=81920 expected_recv=127135 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=81920 expected_recv=131406 expected_overflow=0 exact_recv_per_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=128640 slack_slots=35200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=81920 expected_recv=131406 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=81920 expected_recv=129810 expected_overflow=0 exact_recv_per_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131664 expected_overflow=0 exact_recv_per_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=81920 expected_recv=129810 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131664 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=81920 expected_recv=128883 expected_overflow=0 exact_recv_per_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=81920 expected_recv=128883 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=81920 expected_recv=128682 expected_overflow=0 exact_recv_per_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=81920 expected_recv=128682 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=81920 expected_recv=125291 expected_overflow=0 exact_recv_per_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=81920 expected_recv=125291 actual_valid_recv=126592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131159 expected_overflow=0 exact_recv_per_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=126592 slack_slots=37248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131159 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131231 expected_overflow=0 exact_recv_per_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131357 expected_overflow=0 exact_recv_per_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131231 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131357 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=81920 expected_recv=125971 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=81920 expected_recv=129892 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=81920 expected_recv=130794 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=81920 expected_recv=125971 actual_valid_recv=127616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=81920 expected_recv=129892 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=127616 slack_slots=36224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=81920 expected_recv=130794 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131761 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131761 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131959 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131100 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131959 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=81920 expected_recv=124219 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131100 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=81920 expected_recv=127136 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=81920 expected_recv=124219 actual_valid_recv=125568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=81920 expected_recv=129263 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=81920 expected_recv=127136 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=125568 slack_slots=38272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=81920 expected_recv=129263 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=128512 slack_slots=35328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=81920 expected_recv=130258 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=81920 expected_recv=130258 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=81920 expected_recv=132239 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131413 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=81920 expected_recv=132239 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131709 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131413 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=81920 expected_recv=124282 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131709 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=81920 expected_recv=124282 actual_valid_recv=125568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131780 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=125568 slack_slots=38272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131780 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=81920 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=81920 expected_recv=132387 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=81920 expected_recv=128999 expected_overflow=0 exact_recv_per_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=81920 expected_recv=128999 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=81920 expected_recv=125273 expected_overflow=0 exact_recv_per_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=81920 expected_recv=125273 actual_valid_recv=126848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=81920 expected_recv=126600 expected_overflow=0 exact_recv_per_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=126848 slack_slots=36992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=81920 expected_recv=126600 actual_valid_recv=128256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=128256 slack_slots=35584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=81920 expected_recv=128852 expected_overflow=0 exact_recv_per_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=81920 expected_recv=128852 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131200 expected_overflow=0 exact_recv_per_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=81920 expected_recv=130325 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131200 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=81920 expected_recv=130325 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=81920 expected_recv=128113 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=81920 expected_recv=128113 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=81920 expected_recv=128222 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=81920 expected_recv=128222 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=129664 slack_slots=34176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=81920 expected_recv=125530 expected_overflow=0 exact_recv_per_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=81920 expected_recv=128742 expected_overflow=0 exact_recv_per_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=81920 expected_recv=125530 actual_valid_recv=126848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=81920 expected_recv=128742 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=126848 slack_slots=36992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=81920 expected_recv=131202 expected_overflow=0 exact_recv_per_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=81920 expected_recv=131202 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=81920 expected_recv=123071 expected_overflow=0 exact_recv_per_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=81920 expected_recv=123071 actual_valid_recv=124672 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=81920 expected_recv=130047 expected_overflow=0 exact_recv_per_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=124672 slack_slots=39168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=81920 expected_recv=130047 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=81920 expected_recv=132311 expected_overflow=0 exact_recv_per_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=81920 expected_recv=132311 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=134144 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=81920 expected_recv=126669 expected_overflow=0 exact_recv_per_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=81920 expected_recv=126669 actual_valid_recv=128000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=128000 slack_slots=35840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[56524, 41558, 85588, 67675, 70790, 86238, 43792, 72123] avg=65536.0000 max=86238 max_rank=5 min=41558 min_rank=1 imbalance_ratio=1.315887 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=81920 expected_recv=130012 expected_overflow=0 exact_recv_per_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[56524, 41558, 85588, 67675, 70790, 86238, 43792, 72123] avg=65536.0000 max=86238 max_rank=5 min=41558 min_rank=1 imbalance_ratio=1.315887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[56524, 41558, 85588, 67675, 70790, 86238, 43792, 72123] avg=65536.0000 max=86238 max_rank=5 min=41558 min_rank=1 imbalance_ratio=1.315887 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=81920 expected_recv=130012 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[56524, 41558, 85588, 67675, 70790, 86238, 43792, 72123] avg=65536.0000 max=86238 max_rank=5 min=41558 min_rank=1 imbalance_ratio=1.315887 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] avg=65536.0000 max=66192 max_rank=1 min=63125 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20046 max_load_reduction_pct=0.232450 moved_to_replicas=104080 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[56524, 41558, 85588, 67675, 70790, 86238, 43792, 72123] avg=65536.0000 max=86238 max_rank=5 min=41558 min_rank=1 imbalance_ratio=1.315887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] avg=65536.0000 max=66192 max_rank=1 min=63125 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20046 max_load_reduction_pct=0.232450 moved_to_replicas=104080 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] avg=65536.0000 max=66192 max_rank=1 min=63125 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20046 max_load_reduction_pct=0.232450 moved_to_replicas=104080 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[56524, 41558, 85588, 67675, 70790, 86238, 43792, 72123] avg=65536.0000 max=86238 max_rank=5 min=41558 min_rank=1 imbalance_ratio=1.315887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] avg=65536.0000 max=66192 max_rank=1 min=63125 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20046 max_load_reduction_pct=0.232450 moved_to_replicas=104080 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[56524, 41558, 85588, 67675, 70790, 86238, 43792, 72123] avg=65536.0000 max=86238 max_rank=5 min=41558 min_rank=1 imbalance_ratio=1.315887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[56524, 41558, 85588, 67675, 70790, 86238, 43792, 72123] avg=65536.0000 max=86238 max_rank=5 min=41558 min_rank=1 imbalance_ratio=1.315887 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[34294, 37684, 141733, 44867, 84275, 64897, 49614, 66924] avg=65536.0000 max=141733 max_rank=2 min=34294 min_rank=0 imbalance_ratio=2.162674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] avg=65536.0000 max=66192 max_rank=1 min=63125 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20046 max_load_reduction_pct=0.232450 moved_to_replicas=104080 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] avg=65536.0000 max=66192 max_rank=1 min=63125 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20046 max_load_reduction_pct=0.232450 moved_to_replicas=104080 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] avg=65536.0000 max=66192 max_rank=1 min=63125 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20046 max_load_reduction_pct=0.232450 moved_to_replicas=104080 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[34294, 37684, 141733, 44867, 84275, 64897, 49614, 66924] avg=65536.0000 max=141733 max_rank=2 min=34294 min_rank=0 imbalance_ratio=2.162674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[34294, 37684, 141733, 44867, 84275, 64897, 49614, 66924] avg=65536.0000 max=141733 max_rank=2 min=34294 min_rank=0 imbalance_ratio=2.162674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[34294, 37684, 141733, 44867, 84275, 64897, 49614, 66924] avg=65536.0000 max=141733 max_rank=2 min=34294 min_rank=0 imbalance_ratio=2.162674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62240 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=75540 max_load_reduction_pct=0.532974 moved_to_replicas=200710 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[64011, 66192, 66192, 66192, 66192, 66192, 63125, 66192] avg=65536.0000 max=66192 max_rank=1 min=63125 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=20046 max_load_reduction_pct=0.232450 moved_to_replicas=104080 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62240 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=75540 max_load_reduction_pct=0.532974 moved_to_replicas=200710 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[34294, 37684, 141733, 44867, 84275, 64897, 49614, 66924] avg=65536.0000 max=141733 max_rank=2 min=34294 min_rank=0 imbalance_ratio=2.162674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62240 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=75540 max_load_reduction_pct=0.532974 moved_to_replicas=200710 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[34294, 37684, 141733, 44867, 84275, 64897, 49614, 66924] avg=65536.0000 max=141733 max_rank=2 min=34294 min_rank=0 imbalance_ratio=2.162674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62240 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=75540 max_load_reduction_pct=0.532974 moved_to_replicas=200710 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[34294, 37684, 141733, 44867, 84275, 64897, 49614, 66924] avg=65536.0000 max=141733 max_rank=2 min=34294 min_rank=0 imbalance_ratio=2.162674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62240 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=75540 max_load_reduction_pct=0.532974 moved_to_replicas=200710 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[63247, 110043, 41387, 76570, 75654, 37716, 36725, 82946] avg=65536.0000 max=110043 max_rank=1 min=36725 min_rank=6 imbalance_ratio=1.679123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[34294, 37684, 141733, 44867, 84275, 64897, 49614, 66924] avg=65536.0000 max=141733 max_rank=2 min=34294 min_rank=0 imbalance_ratio=2.162674 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62240 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=75540 max_load_reduction_pct=0.532974 moved_to_replicas=200710 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[63247, 110043, 41387, 76570, 75654, 37716, 36725, 82946] avg=65536.0000 max=110043 max_rank=1 min=36725 min_rank=6 imbalance_ratio=1.679123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62240 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=75540 max_load_reduction_pct=0.532974 moved_to_replicas=200710 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[63247, 110043, 41387, 76570, 75654, 37716, 36725, 82946] avg=65536.0000 max=110043 max_rank=1 min=36725 min_rank=6 imbalance_ratio=1.679123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[63247, 110043, 41387, 76570, 75654, 37716, 36725, 82946] avg=65536.0000 max=110043 max_rank=1 min=36725 min_rank=6 imbalance_ratio=1.679123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63247 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43850 max_load_reduction_pct=0.398481 moved_to_replicas=154418 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[66192, 66192, 66191, 62240, 66193, 64897, 66191, 66192] avg=65536.0000 max=66193 max_rank=4 min=62240 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=75540 max_load_reduction_pct=0.532974 moved_to_replicas=200710 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63247 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43850 max_load_reduction_pct=0.398481 moved_to_replicas=154418 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[63247, 110043, 41387, 76570, 75654, 37716, 36725, 82946] avg=65536.0000 max=110043 max_rank=1 min=36725 min_rank=6 imbalance_ratio=1.679123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63247 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43850 max_load_reduction_pct=0.398481 moved_to_replicas=154418 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63247 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43850 max_load_reduction_pct=0.398481 moved_to_replicas=154418 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[63247, 110043, 41387, 76570, 75654, 37716, 36725, 82946] avg=65536.0000 max=110043 max_rank=1 min=36725 min_rank=6 imbalance_ratio=1.679123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[63247, 110043, 41387, 76570, 75654, 37716, 36725, 82946] avg=65536.0000 max=110043 max_rank=1 min=36725 min_rank=6 imbalance_ratio=1.679123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[74501, 34565, 84037, 66241, 104971, 25069, 83786, 51118] avg=65536.0000 max=104971 max_rank=4 min=25069 min_rank=5 imbalance_ratio=1.601730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[63247, 110043, 41387, 76570, 75654, 37716, 36725, 82946] avg=65536.0000 max=110043 max_rank=1 min=36725 min_rank=6 imbalance_ratio=1.679123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63247 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43850 max_load_reduction_pct=0.398481 moved_to_replicas=154418 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[74501, 34565, 84037, 66241, 104971, 25069, 83786, 51118] avg=65536.0000 max=104971 max_rank=4 min=25069 min_rank=5 imbalance_ratio=1.601730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[74501, 34565, 84037, 66241, 104971, 25069, 83786, 51118] avg=65536.0000 max=104971 max_rank=4 min=25069 min_rank=5 imbalance_ratio=1.601730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63247 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43850 max_load_reduction_pct=0.398481 moved_to_replicas=154418 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[74501, 34565, 84037, 66241, 104971, 25069, 83786, 51118] avg=65536.0000 max=104971 max_rank=4 min=25069 min_rank=5 imbalance_ratio=1.601730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63247 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43850 max_load_reduction_pct=0.398481 moved_to_replicas=154418 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38779 max_load_reduction_pct=0.369426 moved_to_replicas=184668 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[63247, 66192, 66191, 66192, 66192, 63889, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63247 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=43850 max_load_reduction_pct=0.398481 moved_to_replicas=154418 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38779 max_load_reduction_pct=0.369426 moved_to_replicas=184668 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38779 max_load_reduction_pct=0.369426 moved_to_replicas=184668 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38779 max_load_reduction_pct=0.369426 moved_to_replicas=184668 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[74501, 34565, 84037, 66241, 104971, 25069, 83786, 51118] avg=65536.0000 max=104971 max_rank=4 min=25069 min_rank=5 imbalance_ratio=1.601730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[74501, 34565, 84037, 66241, 104971, 25069, 83786, 51118] avg=65536.0000 max=104971 max_rank=4 min=25069 min_rank=5 imbalance_ratio=1.601730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[74501, 34565, 84037, 66241, 104971, 25069, 83786, 51118] avg=65536.0000 max=104971 max_rank=4 min=25069 min_rank=5 imbalance_ratio=1.601730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[26707, 86624, 76069, 38300, 144135, 43794, 63618, 45041] avg=65536.0000 max=144135 max_rank=4 min=26707 min_rank=0 imbalance_ratio=2.199326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38779 max_load_reduction_pct=0.369426 moved_to_replicas=184668 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[74501, 34565, 84037, 66241, 104971, 25069, 83786, 51118] avg=65536.0000 max=104971 max_rank=4 min=25069 min_rank=5 imbalance_ratio=1.601730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[26707, 86624, 76069, 38300, 144135, 43794, 63618, 45041] avg=65536.0000 max=144135 max_rank=4 min=26707 min_rank=0 imbalance_ratio=2.199326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[26707, 86624, 76069, 38300, 144135, 43794, 63618, 45041] avg=65536.0000 max=144135 max_rank=4 min=26707 min_rank=0 imbalance_ratio=2.199326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38779 max_load_reduction_pct=0.369426 moved_to_replicas=184668 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[26707, 86624, 76069, 38300, 144135, 43794, 63618, 45041] avg=65536.0000 max=144135 max_rank=4 min=26707 min_rank=0 imbalance_ratio=2.199326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38779 max_load_reduction_pct=0.369426 moved_to_replicas=184668 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] avg=65536.0000 max=66193 max_rank=0 min=63618 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77942 max_load_reduction_pct=0.540757 moved_to_replicas=251446 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[66192, 60944, 66192, 66192, 66192, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=60944 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=38779 max_load_reduction_pct=0.369426 moved_to_replicas=184668 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] avg=65536.0000 max=66193 max_rank=0 min=63618 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77942 max_load_reduction_pct=0.540757 moved_to_replicas=251446 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] avg=65536.0000 max=66193 max_rank=0 min=63618 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77942 max_load_reduction_pct=0.540757 moved_to_replicas=251446 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] avg=65536.0000 max=66193 max_rank=0 min=63618 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77942 max_load_reduction_pct=0.540757 moved_to_replicas=251446 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[26707, 86624, 76069, 38300, 144135, 43794, 63618, 45041] avg=65536.0000 max=144135 max_rank=4 min=26707 min_rank=0 imbalance_ratio=2.199326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[26707, 86624, 76069, 38300, 144135, 43794, 63618, 45041] avg=65536.0000 max=144135 max_rank=4 min=26707 min_rank=0 imbalance_ratio=2.199326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[26707, 86624, 76069, 38300, 144135, 43794, 63618, 45041] avg=65536.0000 max=144135 max_rank=4 min=26707 min_rank=0 imbalance_ratio=2.199326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[33741, 92398, 103002, 59718, 73375, 48651, 20670, 92733] avg=65536.0000 max=103002 max_rank=2 min=20670 min_rank=6 imbalance_ratio=1.571686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] avg=65536.0000 max=66193 max_rank=0 min=63618 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77942 max_load_reduction_pct=0.540757 moved_to_replicas=251446 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[26707, 86624, 76069, 38300, 144135, 43794, 63618, 45041] avg=65536.0000 max=144135 max_rank=4 min=26707 min_rank=0 imbalance_ratio=2.199326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[33741, 92398, 103002, 59718, 73375, 48651, 20670, 92733] avg=65536.0000 max=103002 max_rank=2 min=20670 min_rank=6 imbalance_ratio=1.571686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[33741, 92398, 103002, 59718, 73375, 48651, 20670, 92733] avg=65536.0000 max=103002 max_rank=2 min=20670 min_rank=6 imbalance_ratio=1.571686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] avg=65536.0000 max=66193 max_rank=0 min=63618 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77942 max_load_reduction_pct=0.540757 moved_to_replicas=251446 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[33741, 92398, 103002, 59718, 73375, 48651, 20670, 92733] avg=65536.0000 max=103002 max_rank=2 min=20670 min_rank=6 imbalance_ratio=1.571686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] avg=65536.0000 max=66193 max_rank=0 min=63618 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77942 max_load_reduction_pct=0.540757 moved_to_replicas=251446 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] avg=65536.0000 max=66193 max_rank=1 min=62167 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36809 max_load_reduction_pct=0.357362 moved_to_replicas=196409 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] avg=65536.0000 max=66193 max_rank=1 min=62167 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36809 max_load_reduction_pct=0.357362 moved_to_replicas=196409 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[66193, 66193, 66191, 66192, 66192, 64237, 63618, 65472] avg=65536.0000 max=66193 max_rank=0 min=63618 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=77942 max_load_reduction_pct=0.540757 moved_to_replicas=251446 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[33741, 92398, 103002, 59718, 73375, 48651, 20670, 92733] avg=65536.0000 max=103002 max_rank=2 min=20670 min_rank=6 imbalance_ratio=1.571686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] avg=65536.0000 max=66193 max_rank=1 min=62167 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36809 max_load_reduction_pct=0.357362 moved_to_replicas=196409 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] avg=65536.0000 max=66193 max_rank=1 min=62167 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36809 max_load_reduction_pct=0.357362 moved_to_replicas=196409 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[33741, 92398, 103002, 59718, 73375, 48651, 20670, 92733] avg=65536.0000 max=103002 max_rank=2 min=20670 min_rank=6 imbalance_ratio=1.571686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[33741, 92398, 103002, 59718, 73375, 48651, 20670, 92733] avg=65536.0000 max=103002 max_rank=2 min=20670 min_rank=6 imbalance_ratio=1.571686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[119504, 3225, 5091, 68456, 35801, 78629, 110156, 103426] avg=65536.0000 max=119504 max_rank=0 min=3225 min_rank=1 imbalance_ratio=1.823486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] avg=65536.0000 max=66193 max_rank=1 min=62167 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36809 max_load_reduction_pct=0.357362 moved_to_replicas=196409 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[119504, 3225, 5091, 68456, 35801, 78629, 110156, 103426] avg=65536.0000 max=119504 max_rank=0 min=3225 min_rank=1 imbalance_ratio=1.823486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[33741, 92398, 103002, 59718, 73375, 48651, 20670, 92733] avg=65536.0000 max=103002 max_rank=2 min=20670 min_rank=6 imbalance_ratio=1.571686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] avg=65536.0000 max=66193 max_rank=1 min=62167 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36809 max_load_reduction_pct=0.357362 moved_to_replicas=196409 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[119504, 3225, 5091, 68456, 35801, 78629, 110156, 103426] avg=65536.0000 max=119504 max_rank=0 min=3225 min_rank=1 imbalance_ratio=1.823486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] avg=65536.0000 max=66193 max_rank=1 min=62167 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36809 max_load_reduction_pct=0.357362 moved_to_replicas=196409 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[119504, 3225, 5091, 68456, 35801, 78629, 110156, 103426] avg=65536.0000 max=119504 max_rank=0 min=3225 min_rank=1 imbalance_ratio=1.823486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53311 max_load_reduction_pct=0.446102 moved_to_replicas=286063 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53311 max_load_reduction_pct=0.446102 moved_to_replicas=286063 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[64967, 66193, 66192, 66191, 66193, 66193, 62167, 66192] avg=65536.0000 max=66193 max_rank=1 min=62167 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=36809 max_load_reduction_pct=0.357362 moved_to_replicas=196409 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53311 max_load_reduction_pct=0.446102 moved_to_replicas=286063 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[119504, 3225, 5091, 68456, 35801, 78629, 110156, 103426] avg=65536.0000 max=119504 max_rank=0 min=3225 min_rank=1 imbalance_ratio=1.823486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53311 max_load_reduction_pct=0.446102 moved_to_replicas=286063 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[119504, 3225, 5091, 68456, 35801, 78629, 110156, 103426] avg=65536.0000 max=119504 max_rank=0 min=3225 min_rank=1 imbalance_ratio=1.823486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[119504, 3225, 5091, 68456, 35801, 78629, 110156, 103426] avg=65536.0000 max=119504 max_rank=0 min=3225 min_rank=1 imbalance_ratio=1.823486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[100171, 2572, 76006, 58449, 125870, 47664, 5869, 107687] avg=65536.0000 max=125870 max_rank=4 min=2572 min_rank=1 imbalance_ratio=1.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53311 max_load_reduction_pct=0.446102 moved_to_replicas=286063 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[100171, 2572, 76006, 58449, 125870, 47664, 5869, 107687] avg=65536.0000 max=125870 max_rank=4 min=2572 min_rank=1 imbalance_ratio=1.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[119504, 3225, 5091, 68456, 35801, 78629, 110156, 103426] avg=65536.0000 max=119504 max_rank=0 min=3225 min_rank=1 imbalance_ratio=1.823486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53311 max_load_reduction_pct=0.446102 moved_to_replicas=286063 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[100171, 2572, 76006, 58449, 125870, 47664, 5869, 107687] avg=65536.0000 max=125870 max_rank=4 min=2572 min_rank=1 imbalance_ratio=1.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53311 max_load_reduction_pct=0.446102 moved_to_replicas=286063 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[100171, 2572, 76006, 58449, 125870, 47664, 5869, 107687] avg=65536.0000 max=125870 max_rank=4 min=2572 min_rank=1 imbalance_ratio=1.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] avg=65536.0000 max=66192 max_rank=0 min=61566 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=59678 max_load_reduction_pct=0.474124 moved_to_replicas=270937 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] avg=65536.0000 max=66192 max_rank=0 min=61566 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=59678 max_load_reduction_pct=0.474124 moved_to_replicas=270937 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] avg=65536.0000 max=66192 max_rank=0 min=61566 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=59678 max_load_reduction_pct=0.474124 moved_to_replicas=270937 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[66192, 66193, 60944, 66192, 66192, 66192, 66191, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=53311 max_load_reduction_pct=0.446102 moved_to_replicas=286063 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] avg=65536.0000 max=66192 max_rank=0 min=61566 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=59678 max_load_reduction_pct=0.474124 moved_to_replicas=270937 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[100171, 2572, 76006, 58449, 125870, 47664, 5869, 107687] avg=65536.0000 max=125870 max_rank=4 min=2572 min_rank=1 imbalance_ratio=1.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[100171, 2572, 76006, 58449, 125870, 47664, 5869, 107687] avg=65536.0000 max=125870 max_rank=4 min=2572 min_rank=1 imbalance_ratio=1.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[100171, 2572, 76006, 58449, 125870, 47664, 5869, 107687] avg=65536.0000 max=125870 max_rank=4 min=2572 min_rank=1 imbalance_ratio=1.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[85020, 31027, 82996, 92252, 79332, 83882, 57871, 11908] avg=65536.0000 max=92252 max_rank=3 min=11908 min_rank=7 imbalance_ratio=1.407654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[85020, 31027, 82996, 92252, 79332, 83882, 57871, 11908] avg=65536.0000 max=92252 max_rank=3 min=11908 min_rank=7 imbalance_ratio=1.407654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[85020, 31027, 82996, 92252, 79332, 83882, 57871, 11908] avg=65536.0000 max=92252 max_rank=3 min=11908 min_rank=7 imbalance_ratio=1.407654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] avg=65536.0000 max=66192 max_rank=0 min=61566 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=59678 max_load_reduction_pct=0.474124 moved_to_replicas=270937 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] avg=65536.0000 max=66192 max_rank=0 min=61566 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=59678 max_load_reduction_pct=0.474124 moved_to_replicas=270937 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[85020, 31027, 82996, 92252, 79332, 83882, 57871, 11908] avg=65536.0000 max=92252 max_rank=3 min=11908 min_rank=7 imbalance_ratio=1.407654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[100171, 2572, 76006, 58449, 125870, 47664, 5869, 107687] avg=65536.0000 max=125870 max_rank=4 min=2572 min_rank=1 imbalance_ratio=1.920624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] avg=65536.0000 max=66192 max_rank=0 min=61566 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=59678 max_load_reduction_pct=0.474124 moved_to_replicas=270937 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26059 max_load_reduction_pct=0.282476 moved_to_replicas=199386 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26059 max_load_reduction_pct=0.282476 moved_to_replicas=199386 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26059 max_load_reduction_pct=0.282476 moved_to_replicas=199386 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26059 max_load_reduction_pct=0.282476 moved_to_replicas=199386 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 66192, 61566, 65570, 66192] avg=65536.0000 max=66192 max_rank=0 min=61566 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=59678 max_load_reduction_pct=0.474124 moved_to_replicas=270937 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[85020, 31027, 82996, 92252, 79332, 83882, 57871, 11908] avg=65536.0000 max=92252 max_rank=3 min=11908 min_rank=7 imbalance_ratio=1.407654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[85020, 31027, 82996, 92252, 79332, 83882, 57871, 11908] avg=65536.0000 max=92252 max_rank=3 min=11908 min_rank=7 imbalance_ratio=1.407654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[85020, 31027, 82996, 92252, 79332, 83882, 57871, 11908] avg=65536.0000 max=92252 max_rank=3 min=11908 min_rank=7 imbalance_ratio=1.407654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[66625, 36033, 60983, 118171, 109213, 33874, 33871, 65518] avg=65536.0000 max=118171 max_rank=3 min=33871 min_rank=6 imbalance_ratio=1.803146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[66625, 36033, 60983, 118171, 109213, 33874, 33871, 65518] avg=65536.0000 max=118171 max_rank=3 min=33871 min_rank=6 imbalance_ratio=1.803146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[66625, 36033, 60983, 118171, 109213, 33874, 33871, 65518] avg=65536.0000 max=118171 max_rank=3 min=33871 min_rank=6 imbalance_ratio=1.803146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26059 max_load_reduction_pct=0.282476 moved_to_replicas=199386 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26059 max_load_reduction_pct=0.282476 moved_to_replicas=199386 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[66625, 36033, 60983, 118171, 109213, 33874, 33871, 65518] avg=65536.0000 max=118171 max_rank=3 min=33871 min_rank=6 imbalance_ratio=1.803146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[85020, 31027, 82996, 92252, 79332, 83882, 57871, 11908] avg=65536.0000 max=92252 max_rank=3 min=11908 min_rank=7 imbalance_ratio=1.407654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26059 max_load_reduction_pct=0.282476 moved_to_replicas=199386 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] avg=65536.0000 max=66192 max_rank=0 min=61763 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=51979 max_load_reduction_pct=0.439863 moved_to_replicas=197424 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] avg=65536.0000 max=66192 max_rank=0 min=61763 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=51979 max_load_reduction_pct=0.439863 moved_to_replicas=197424 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] avg=65536.0000 max=66192 max_rank=0 min=61763 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=51979 max_load_reduction_pct=0.439863 moved_to_replicas=197424 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] avg=65536.0000 max=66192 max_rank=0 min=61763 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=51979 max_load_reduction_pct=0.439863 moved_to_replicas=197424 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[66192, 66193, 66191, 66193, 66192, 66191, 60944, 66192] avg=65536.0000 max=66193 max_rank=1 min=60944 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=26059 max_load_reduction_pct=0.282476 moved_to_replicas=199386 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[66625, 36033, 60983, 118171, 109213, 33874, 33871, 65518] avg=65536.0000 max=118171 max_rank=3 min=33871 min_rank=6 imbalance_ratio=1.803146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[66625, 36033, 60983, 118171, 109213, 33874, 33871, 65518] avg=65536.0000 max=118171 max_rank=3 min=33871 min_rank=6 imbalance_ratio=1.803146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[66625, 36033, 60983, 118171, 109213, 33874, 33871, 65518] avg=65536.0000 max=118171 max_rank=3 min=33871 min_rank=6 imbalance_ratio=1.803146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[69693, 31794, 83896, 31906, 105976, 39500, 118545, 42978] avg=65536.0000 max=118545 max_rank=6 min=31794 min_rank=1 imbalance_ratio=1.808853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[69693, 31794, 83896, 31906, 105976, 39500, 118545, 42978] avg=65536.0000 max=118545 max_rank=6 min=31794 min_rank=1 imbalance_ratio=1.808853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[69693, 31794, 83896, 31906, 105976, 39500, 118545, 42978] avg=65536.0000 max=118545 max_rank=6 min=31794 min_rank=1 imbalance_ratio=1.808853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] avg=65536.0000 max=66192 max_rank=0 min=61763 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=51979 max_load_reduction_pct=0.439863 moved_to_replicas=197424 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] avg=65536.0000 max=66192 max_rank=0 min=61763 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=51979 max_load_reduction_pct=0.439863 moved_to_replicas=197424 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[69693, 31794, 83896, 31906, 105976, 39500, 118545, 42978] avg=65536.0000 max=118545 max_rank=6 min=31794 min_rank=1 imbalance_ratio=1.808853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] avg=65536.0000 max=66192 max_rank=0 min=61763 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=51979 max_load_reduction_pct=0.439863 moved_to_replicas=197424 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[66625, 36033, 60983, 118171, 109213, 33874, 33871, 65518] avg=65536.0000 max=118171 max_rank=3 min=33871 min_rank=6 imbalance_ratio=1.803146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] avg=65536.0000 max=66193 max_rank=5 min=62806 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52352 max_load_reduction_pct=0.441621 moved_to_replicas=230918 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] avg=65536.0000 max=66193 max_rank=5 min=62806 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52352 max_load_reduction_pct=0.441621 moved_to_replicas=230918 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] avg=65536.0000 max=66193 max_rank=5 min=62806 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52352 max_load_reduction_pct=0.441621 moved_to_replicas=230918 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] avg=65536.0000 max=66193 max_rank=5 min=62806 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52352 max_load_reduction_pct=0.441621 moved_to_replicas=230918 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[66192, 66192, 61763, 66192, 66192, 66192, 66047, 65518] avg=65536.0000 max=66192 max_rank=0 min=61763 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=51979 max_load_reduction_pct=0.439863 moved_to_replicas=197424 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[69693, 31794, 83896, 31906, 105976, 39500, 118545, 42978] avg=65536.0000 max=118545 max_rank=6 min=31794 min_rank=1 imbalance_ratio=1.808853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[69693, 31794, 83896, 31906, 105976, 39500, 118545, 42978] avg=65536.0000 max=118545 max_rank=6 min=31794 min_rank=1 imbalance_ratio=1.808853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[69693, 31794, 83896, 31906, 105976, 39500, 118545, 42978] avg=65536.0000 max=118545 max_rank=6 min=31794 min_rank=1 imbalance_ratio=1.808853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[81476, 20828, 94114, 91286, 69546, 46370, 7568, 113100] avg=65536.0000 max=113100 max_rank=7 min=7568 min_rank=6 imbalance_ratio=1.725769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] avg=65536.0000 max=66193 max_rank=5 min=62806 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52352 max_load_reduction_pct=0.441621 moved_to_replicas=230918 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] avg=65536.0000 max=66193 max_rank=5 min=62806 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52352 max_load_reduction_pct=0.441621 moved_to_replicas=230918 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[81476, 20828, 94114, 91286, 69546, 46370, 7568, 113100] avg=65536.0000 max=113100 max_rank=7 min=7568 min_rank=6 imbalance_ratio=1.725769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[81476, 20828, 94114, 91286, 69546, 46370, 7568, 113100] avg=65536.0000 max=113100 max_rank=7 min=7568 min_rank=6 imbalance_ratio=1.725769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] avg=65536.0000 max=66193 max_rank=5 min=62806 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52352 max_load_reduction_pct=0.441621 moved_to_replicas=230918 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[81476, 20828, 94114, 91286, 69546, 46370, 7568, 113100] avg=65536.0000 max=113100 max_rank=7 min=7568 min_rank=6 imbalance_ratio=1.725769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[69693, 31794, 83896, 31906, 105976, 39500, 118545, 42978] avg=65536.0000 max=118545 max_rank=6 min=31794 min_rank=1 imbalance_ratio=1.808853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] avg=65536.0000 max=66193 max_rank=2 min=63760 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=46907 max_load_reduction_pct=0.414739 moved_to_replicas=241351 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] avg=65536.0000 max=66193 max_rank=2 min=63760 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=46907 max_load_reduction_pct=0.414739 moved_to_replicas=241351 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] avg=65536.0000 max=66193 max_rank=2 min=63760 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=46907 max_load_reduction_pct=0.414739 moved_to_replicas=241351 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] avg=65536.0000 max=66193 max_rank=2 min=63760 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=46907 max_load_reduction_pct=0.414739 moved_to_replicas=241351 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[81476, 20828, 94114, 91286, 69546, 46370, 7568, 113100] avg=65536.0000 max=113100 max_rank=7 min=7568 min_rank=6 imbalance_ratio=1.725769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[66192, 64329, 66192, 66192, 66192, 66193, 66192, 62806] avg=65536.0000 max=66193 max_rank=5 min=62806 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=52352 max_load_reduction_pct=0.441621 moved_to_replicas=230918 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[81476, 20828, 94114, 91286, 69546, 46370, 7568, 113100] avg=65536.0000 max=113100 max_rank=7 min=7568 min_rank=6 imbalance_ratio=1.725769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[81476, 20828, 94114, 91286, 69546, 46370, 7568, 113100] avg=65536.0000 max=113100 max_rank=7 min=7568 min_rank=6 imbalance_ratio=1.725769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[57830, 116059, 14077, 892, 74874, 70758, 112999, 76799] avg=65536.0000 max=116059 max_rank=1 min=892 min_rank=3 imbalance_ratio=1.770920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] avg=65536.0000 max=66193 max_rank=2 min=63760 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=46907 max_load_reduction_pct=0.414739 moved_to_replicas=241351 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] avg=65536.0000 max=66193 max_rank=2 min=63760 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=46907 max_load_reduction_pct=0.414739 moved_to_replicas=241351 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[57830, 116059, 14077, 892, 74874, 70758, 112999, 76799] avg=65536.0000 max=116059 max_rank=1 min=892 min_rank=3 imbalance_ratio=1.770920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[57830, 116059, 14077, 892, 74874, 70758, 112999, 76799] avg=65536.0000 max=116059 max_rank=1 min=892 min_rank=3 imbalance_ratio=1.770920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[57830, 116059, 14077, 892, 74874, 70758, 112999, 76799] avg=65536.0000 max=116059 max_rank=1 min=892 min_rank=3 imbalance_ratio=1.770920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] avg=65536.0000 max=66193 max_rank=2 min=63760 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=46907 max_load_reduction_pct=0.414739 moved_to_replicas=241351 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62994 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49866 max_load_reduction_pct=0.429661 moved_to_replicas=243409 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[81476, 20828, 94114, 91286, 69546, 46370, 7568, 113100] avg=65536.0000 max=113100 max_rank=7 min=7568 min_rank=6 imbalance_ratio=1.725769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62994 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49866 max_load_reduction_pct=0.429661 moved_to_replicas=243409 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62994 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49866 max_load_reduction_pct=0.429661 moved_to_replicas=243409 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62994 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49866 max_load_reduction_pct=0.429661 moved_to_replicas=243409 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[57830, 116059, 14077, 892, 74874, 70758, 112999, 76799] avg=65536.0000 max=116059 max_rank=1 min=892 min_rank=3 imbalance_ratio=1.770920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[66192, 64559, 66193, 66192, 66192, 65008, 63760, 66192] avg=65536.0000 max=66193 max_rank=2 min=63760 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=46907 max_load_reduction_pct=0.414739 moved_to_replicas=241351 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[57830, 116059, 14077, 892, 74874, 70758, 112999, 76799] avg=65536.0000 max=116059 max_rank=1 min=892 min_rank=3 imbalance_ratio=1.770920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[57830, 116059, 14077, 892, 74874, 70758, 112999, 76799] avg=65536.0000 max=116059 max_rank=1 min=892 min_rank=3 imbalance_ratio=1.770920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[91342, 33106, 95198, 11962, 64358, 32779, 115644, 79899] avg=65536.0000 max=115644 max_rank=6 min=11962 min_rank=3 imbalance_ratio=1.764587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62994 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49866 max_load_reduction_pct=0.429661 moved_to_replicas=243409 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[91342, 33106, 95198, 11962, 64358, 32779, 115644, 79899] avg=65536.0000 max=115644 max_rank=6 min=11962 min_rank=3 imbalance_ratio=1.764587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62994 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49866 max_load_reduction_pct=0.429661 moved_to_replicas=243409 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[91342, 33106, 95198, 11962, 64358, 32779, 115644, 79899] avg=65536.0000 max=115644 max_rank=6 min=11962 min_rank=3 imbalance_ratio=1.764587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[91342, 33106, 95198, 11962, 64358, 32779, 115644, 79899] avg=65536.0000 max=115644 max_rank=6 min=11962 min_rank=3 imbalance_ratio=1.764587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62994 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49866 max_load_reduction_pct=0.429661 moved_to_replicas=243409 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=62778 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=49451 max_load_reduction_pct=0.427614 moved_to_replicas=257371 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[57830, 116059, 14077, 892, 74874, 70758, 112999, 76799] avg=65536.0000 max=116059 max_rank=1 min=892 min_rank=3 imbalance_ratio=1.770920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=62778 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=49451 max_load_reduction_pct=0.427614 moved_to_replicas=257371 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=62778 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=49451 max_load_reduction_pct=0.427614 moved_to_replicas=257371 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=62778 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=49451 max_load_reduction_pct=0.427614 moved_to_replicas=257371 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[91342, 33106, 95198, 11962, 64358, 32779, 115644, 79899] avg=65536.0000 max=115644 max_rank=6 min=11962 min_rank=3 imbalance_ratio=1.764587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[91342, 33106, 95198, 11962, 64358, 32779, 115644, 79899] avg=65536.0000 max=115644 max_rank=6 min=11962 min_rank=3 imbalance_ratio=1.764587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[66192, 66191, 62994, 64143, 66192, 66193, 66191, 66192] avg=65536.0000 max=66193 max_rank=5 min=62994 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=49866 max_load_reduction_pct=0.429661 moved_to_replicas=243409 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[91342, 33106, 95198, 11962, 64358, 32779, 115644, 79899] avg=65536.0000 max=115644 max_rank=6 min=11962 min_rank=3 imbalance_ratio=1.764587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[74711, 50798, 91922, 36867, 38401, 88678, 103483, 39428] avg=65536.0000 max=103483 max_rank=6 min=36867 min_rank=3 imbalance_ratio=1.579025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[74711, 50798, 91922, 36867, 38401, 88678, 103483, 39428] avg=65536.0000 max=103483 max_rank=6 min=36867 min_rank=3 imbalance_ratio=1.579025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=62778 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=49451 max_load_reduction_pct=0.427614 moved_to_replicas=257371 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=62778 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=49451 max_load_reduction_pct=0.427614 moved_to_replicas=257371 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[74711, 50798, 91922, 36867, 38401, 88678, 103483, 39428] avg=65536.0000 max=103483 max_rank=6 min=36867 min_rank=3 imbalance_ratio=1.579025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] avg=65536.0000 max=66194 max_rank=2 min=60945 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37289 max_load_reduction_pct=0.360339 moved_to_replicas=190725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=62778 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=49451 max_load_reduction_pct=0.427614 moved_to_replicas=257371 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[74711, 50798, 91922, 36867, 38401, 88678, 103483, 39428] avg=65536.0000 max=103483 max_rank=6 min=36867 min_rank=3 imbalance_ratio=1.579025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[91342, 33106, 95198, 11962, 64358, 32779, 115644, 79899] avg=65536.0000 max=115644 max_rank=6 min=11962 min_rank=3 imbalance_ratio=1.764587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] avg=65536.0000 max=66194 max_rank=2 min=60945 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37289 max_load_reduction_pct=0.360339 moved_to_replicas=190725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] avg=65536.0000 max=66194 max_rank=2 min=60945 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37289 max_load_reduction_pct=0.360339 moved_to_replicas=190725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[74711, 50798, 91922, 36867, 38401, 88678, 103483, 39428] avg=65536.0000 max=103483 max_rank=6 min=36867 min_rank=3 imbalance_ratio=1.579025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] avg=65536.0000 max=66194 max_rank=2 min=60945 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37289 max_load_reduction_pct=0.360339 moved_to_replicas=190725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[74711, 50798, 91922, 36867, 38401, 88678, 103483, 39428] avg=65536.0000 max=103483 max_rank=6 min=36867 min_rank=3 imbalance_ratio=1.579025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[66191, 62778, 66192, 66193, 64358, 66192, 66192, 66192] avg=65536.0000 max=66193 max_rank=3 min=62778 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=49451 max_load_reduction_pct=0.427614 moved_to_replicas=257371 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[23186, 55062, 136038, 73909, 57487, 36563, 61902, 80141] avg=65536.0000 max=136038 max_rank=2 min=23186 min_rank=0 imbalance_ratio=2.075775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[74711, 50798, 91922, 36867, 38401, 88678, 103483, 39428] avg=65536.0000 max=103483 max_rank=6 min=36867 min_rank=3 imbalance_ratio=1.579025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[23186, 55062, 136038, 73909, 57487, 36563, 61902, 80141] avg=65536.0000 max=136038 max_rank=2 min=23186 min_rank=0 imbalance_ratio=2.075775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] avg=65536.0000 max=66194 max_rank=2 min=60945 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37289 max_load_reduction_pct=0.360339 moved_to_replicas=190725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] avg=65536.0000 max=66194 max_rank=2 min=60945 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37289 max_load_reduction_pct=0.360339 moved_to_replicas=190725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[23186, 55062, 136038, 73909, 57487, 36563, 61902, 80141] avg=65536.0000 max=136038 max_rank=2 min=23186 min_rank=0 imbalance_ratio=2.075775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] avg=65536.0000 max=66193 max_rank=2 min=63283 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69845 max_load_reduction_pct=0.513423 moved_to_replicas=182314 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[23186, 55062, 136038, 73909, 57487, 36563, 61902, 80141] avg=65536.0000 max=136038 max_rank=2 min=23186 min_rank=0 imbalance_ratio=2.075775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] avg=65536.0000 max=66194 max_rank=2 min=60945 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37289 max_load_reduction_pct=0.360339 moved_to_replicas=190725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[74711, 50798, 91922, 36867, 38401, 88678, 103483, 39428] avg=65536.0000 max=103483 max_rank=6 min=36867 min_rank=3 imbalance_ratio=1.579025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] avg=65536.0000 max=66193 max_rank=2 min=63283 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69845 max_load_reduction_pct=0.513423 moved_to_replicas=182314 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] avg=65536.0000 max=66193 max_rank=2 min=63283 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69845 max_load_reduction_pct=0.513423 moved_to_replicas=182314 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[23186, 55062, 136038, 73909, 57487, 36563, 61902, 80141] avg=65536.0000 max=136038 max_rank=2 min=23186 min_rank=0 imbalance_ratio=2.075775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] avg=65536.0000 max=66193 max_rank=2 min=63283 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69845 max_load_reduction_pct=0.513423 moved_to_replicas=182314 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[23186, 55062, 136038, 73909, 57487, 36563, 61902, 80141] avg=65536.0000 max=136038 max_rank=2 min=23186 min_rank=0 imbalance_ratio=2.075775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[66191, 60945, 66194, 66192, 66190, 66193, 66192, 66191] avg=65536.0000 max=66194 max_rank=2 min=60945 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=37289 max_load_reduction_pct=0.360339 moved_to_replicas=190725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] avg=65536.0000 max=66193 max_rank=2 min=63283 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69845 max_load_reduction_pct=0.513423 moved_to_replicas=182314 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[23186, 55062, 136038, 73909, 57487, 36563, 61902, 80141] avg=65536.0000 max=136038 max_rank=2 min=23186 min_rank=0 imbalance_ratio=2.075775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] avg=65536.0000 max=66193 max_rank=2 min=63283 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69845 max_load_reduction_pct=0.513423 moved_to_replicas=182314 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[23186, 55062, 136038, 73909, 57487, 36563, 61902, 80141] avg=65536.0000 max=136038 max_rank=2 min=23186 min_rank=0 imbalance_ratio=2.075775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] avg=65536.0000 max=66193 max_rank=2 min=63283 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69845 max_load_reduction_pct=0.513423 moved_to_replicas=182314 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[66190, 63283, 66193, 66193, 66192, 66192, 63854, 66191] avg=65536.0000 max=66193 max_rank=2 min=63283 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=69845 max_load_reduction_pct=0.513423 moved_to_replicas=182314 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=2088503 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=512384 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=2104537 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=496768 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=2105510 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=496256 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=2107645 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=495104 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=2099353 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=503040 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=2078837 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=520448 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=2100529 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=501120 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=2092302 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508288 peak_demand=66195 required_capacity=1.010056 + [2026-08-30 05:14:09] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 550.7 | throughput per GPU (TFLOP/s/GPU): 605.4 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.051981E+01 | loss scale: 1.0 | grad norm: 215.096 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=81920 expected_recv=129400 expected_overflow=0 exact_recv_per_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=81920 expected_recv=127225 expected_overflow=0 exact_recv_per_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=81920 expected_recv=129400 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=81920 expected_recv=127225 actual_valid_recv=128128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=128128 slack_slots=35712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=81920 expected_recv=130030 expected_overflow=0 exact_recv_per_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=81920 expected_recv=130030 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130849 expected_overflow=0 exact_recv_per_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130849 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130399 expected_overflow=0 exact_recv_per_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=81920 expected_recv=128578 expected_overflow=0 exact_recv_per_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130643 expected_overflow=0 exact_recv_per_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130399 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=81920 expected_recv=128578 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130956 expected_overflow=0 exact_recv_per_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130643 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130956 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=81920 expected_recv=128381 expected_overflow=0 exact_recv_per_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=81920 expected_recv=128381 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131145 expected_overflow=0 exact_recv_per_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=81920 expected_recv=129778 expected_overflow=0 exact_recv_per_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131145 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=81920 expected_recv=129778 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=81920 expected_recv=129734 expected_overflow=0 exact_recv_per_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131704 expected_overflow=0 exact_recv_per_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=81920 expected_recv=129734 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131704 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131067 expected_overflow=0 exact_recv_per_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=81920 expected_recv=126184 expected_overflow=0 exact_recv_per_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131067 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=81920 expected_recv=126184 actual_valid_recv=127488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131166 expected_overflow=0 exact_recv_per_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=134272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=127488 slack_slots=36352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131305 expected_overflow=0 exact_recv_per_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=134272 slack_slots=29568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131166 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131533 expected_overflow=0 exact_recv_per_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131305 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131533 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131928 expected_overflow=0 exact_recv_per_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=81920 expected_recv=127659 expected_overflow=0 exact_recv_per_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131928 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=81920 expected_recv=127921 expected_overflow=0 exact_recv_per_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=81920 expected_recv=127921 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=81920 expected_recv=127659 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=129408 slack_slots=34432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131898 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131898 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131680 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=81920 expected_recv=127556 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131680 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=81920 expected_recv=127906 expected_overflow=0 exact_recv_per_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131442 expected_overflow=0 exact_recv_per_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=81920 expected_recv=127556 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=81920 expected_recv=127906 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131442 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=129024 slack_slots=34816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=129408 slack_slots=34432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131008 expected_overflow=0 exact_recv_per_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131008 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=81920 expected_recv=124206 expected_overflow=0 exact_recv_per_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=81920 expected_recv=124206 actual_valid_recv=125824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=125824 slack_slots=38016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=81920 expected_recv=131471 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=81920 expected_recv=130769 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=81920 expected_recv=124417 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=81920 expected_recv=131471 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=81920 expected_recv=130769 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=81920 expected_recv=124417 actual_valid_recv=126336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=81920 expected_recv=129590 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=126336 slack_slots=37504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=81920 expected_recv=129590 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=81920 expected_recv=127751 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=81920 expected_recv=127751 actual_valid_recv=129280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=129280 slack_slots=34560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131085 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131085 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=81920 expected_recv=132378 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=81920 expected_recv=130614 expected_overflow=0 exact_recv_per_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=81920 expected_recv=132378 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=81920 expected_recv=130614 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131457 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131457 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=81920 expected_recv=129593 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=81920 expected_recv=129593 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=81920 expected_recv=126120 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=81920 expected_recv=126120 actual_valid_recv=127488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131877 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131316 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=127488 slack_slots=36352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131877 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131316 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=81920 expected_recv=129799 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=81920 expected_recv=129799 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=81920 expected_recv=130693 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=81920 expected_recv=130693 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131282 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131282 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=81920 expected_recv=130122 expected_overflow=0 exact_recv_per_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=81920 expected_recv=128336 expected_overflow=0 exact_recv_per_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=81920 expected_recv=130122 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=81920 expected_recv=128059 expected_overflow=0 exact_recv_per_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=81920 expected_recv=128336 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=81920 expected_recv=128059 actual_valid_recv=129664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131279 expected_overflow=0 exact_recv_per_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=129664 slack_slots=34176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131279 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=81920 expected_recv=129580 expected_overflow=0 exact_recv_per_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=81920 expected_recv=129580 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=81920 expected_recv=127684 expected_overflow=0 exact_recv_per_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=81920 expected_recv=127684 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=129024 slack_slots=34816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130459 expected_overflow=0 exact_recv_per_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=81920 expected_recv=131749 expected_overflow=0 exact_recv_per_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130459 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=81920 expected_recv=131749 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=81920 expected_recv=129147 expected_overflow=0 exact_recv_per_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=81920 expected_recv=129689 expected_overflow=0 exact_recv_per_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=81920 expected_recv=129147 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=81920 expected_recv=130520 expected_overflow=0 exact_recv_per_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=81920 expected_recv=129689 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=81920 expected_recv=130520 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=81920 expected_recv=128945 expected_overflow=0 exact_recv_per_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=81920 expected_recv=128945 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=81920 expected_recv=128932 expected_overflow=0 exact_recv_per_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=81920 expected_recv=130057 expected_overflow=0 exact_recv_per_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=81920 expected_recv=129886 expected_overflow=0 exact_recv_per_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=81920 expected_recv=130173 expected_overflow=0 exact_recv_per_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=81920 expected_recv=128932 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=81920 expected_recv=130057 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=81920 expected_recv=129886 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=81920 expected_recv=130173 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131238 expected_overflow=0 exact_recv_per_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131238 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131025 expected_overflow=0 exact_recv_per_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=81920 expected_recv=129328 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131025 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=81920 expected_recv=129328 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=81920 expected_recv=130192 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=81920 expected_recv=130192 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=81920 expected_recv=129703 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=81920 expected_recv=129703 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=81920 expected_recv=129817 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[47111, 39136, 99674, 57193, 63838, 96308, 49557, 71471] avg=65536.0000 max=99674 max_rank=2 min=39136 min_rank=1 imbalance_ratio=1.520905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[47111, 39136, 99674, 57193, 63838, 96308, 49557, 71471] avg=65536.0000 max=99674 max_rank=2 min=39136 min_rank=1 imbalance_ratio=1.520905 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=81920 expected_recv=129817 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[47111, 39136, 99674, 57193, 63838, 96308, 49557, 71471] avg=65536.0000 max=99674 max_rank=2 min=39136 min_rank=1 imbalance_ratio=1.520905 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[47111, 39136, 99674, 57193, 63838, 96308, 49557, 71471] avg=65536.0000 max=99674 max_rank=2 min=39136 min_rank=1 imbalance_ratio=1.520905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[47111, 39136, 99674, 57193, 63838, 96308, 49557, 71471] avg=65536.0000 max=99674 max_rank=2 min=39136 min_rank=1 imbalance_ratio=1.520905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=2 min=63645 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33481 max_load_reduction_pct=0.335905 moved_to_replicas=135297 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=2 min=63645 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33481 max_load_reduction_pct=0.335905 moved_to_replicas=135297 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=2 min=63645 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33481 max_load_reduction_pct=0.335905 moved_to_replicas=135297 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[47111, 39136, 99674, 57193, 63838, 96308, 49557, 71471] avg=65536.0000 max=99674 max_rank=2 min=39136 min_rank=1 imbalance_ratio=1.520905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=2 min=63645 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33481 max_load_reduction_pct=0.335905 moved_to_replicas=135297 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=2 min=63645 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33481 max_load_reduction_pct=0.335905 moved_to_replicas=135297 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[47111, 39136, 99674, 57193, 63838, 96308, 49557, 71471] avg=65536.0000 max=99674 max_rank=2 min=39136 min_rank=1 imbalance_ratio=1.520905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[47111, 39136, 99674, 57193, 63838, 96308, 49557, 71471] avg=65536.0000 max=99674 max_rank=2 min=39136 min_rank=1 imbalance_ratio=1.520905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32660, 162476, 113374, 62743, 32576, 21619, 63908, 34932] avg=65536.0000 max=162476 max_rank=1 min=21619 min_rank=5 imbalance_ratio=2.479187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=2 min=63645 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33481 max_load_reduction_pct=0.335905 moved_to_replicas=135297 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32660, 162476, 113374, 62743, 32576, 21619, 63908, 34932] avg=65536.0000 max=162476 max_rank=1 min=21619 min_rank=5 imbalance_ratio=2.479187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32660, 162476, 113374, 62743, 32576, 21619, 63908, 34932] avg=65536.0000 max=162476 max_rank=1 min=21619 min_rank=5 imbalance_ratio=2.479187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=2 min=63645 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33481 max_load_reduction_pct=0.335905 moved_to_replicas=135297 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[65845, 66192, 66193, 63645, 63838, 66192, 66192, 66191] avg=65536.0000 max=66193 max_rank=2 min=63645 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=33481 max_load_reduction_pct=0.335905 moved_to_replicas=135297 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[32660, 162476, 113374, 62743, 32576, 21619, 63908, 34932] avg=65536.0000 max=162476 max_rank=1 min=21619 min_rank=5 imbalance_ratio=2.479187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32660, 162476, 113374, 62743, 32576, 21619, 63908, 34932] avg=65536.0000 max=162476 max_rank=1 min=21619 min_rank=5 imbalance_ratio=2.479187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] avg=65536.0000 max=66192 max_rank=1 min=63908 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96284 max_load_reduction_pct=0.592604 moved_to_replicas=295934 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] avg=65536.0000 max=66192 max_rank=1 min=63908 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96284 max_load_reduction_pct=0.592604 moved_to_replicas=295934 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] avg=65536.0000 max=66192 max_rank=1 min=63908 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96284 max_load_reduction_pct=0.592604 moved_to_replicas=295934 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32660, 162476, 113374, 62743, 32576, 21619, 63908, 34932] avg=65536.0000 max=162476 max_rank=1 min=21619 min_rank=5 imbalance_ratio=2.479187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] avg=65536.0000 max=66192 max_rank=1 min=63908 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96284 max_load_reduction_pct=0.592604 moved_to_replicas=295934 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] avg=65536.0000 max=66192 max_rank=1 min=63908 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96284 max_load_reduction_pct=0.592604 moved_to_replicas=295934 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[32660, 162476, 113374, 62743, 32576, 21619, 63908, 34932] avg=65536.0000 max=162476 max_rank=1 min=21619 min_rank=5 imbalance_ratio=2.479187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32660, 162476, 113374, 62743, 32576, 21619, 63908, 34932] avg=65536.0000 max=162476 max_rank=1 min=21619 min_rank=5 imbalance_ratio=2.479187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] avg=65536.0000 max=66192 max_rank=1 min=63908 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96284 max_load_reduction_pct=0.592604 moved_to_replicas=295934 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[1366, 100574, 65564, 66281, 72793, 61377, 68312, 88021] avg=65536.0000 max=100574 max_rank=1 min=1366 min_rank=0 imbalance_ratio=1.534637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[1366, 100574, 65564, 66281, 72793, 61377, 68312, 88021] avg=65536.0000 max=100574 max_rank=1 min=1366 min_rank=0 imbalance_ratio=1.534637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[1366, 100574, 65564, 66281, 72793, 61377, 68312, 88021] avg=65536.0000 max=100574 max_rank=1 min=1366 min_rank=0 imbalance_ratio=1.534637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] avg=65536.0000 max=66192 max_rank=1 min=63908 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96284 max_load_reduction_pct=0.592604 moved_to_replicas=295934 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[64656, 66192, 66192, 66192, 64764, 66192, 63908, 66192] avg=65536.0000 max=66192 max_rank=1 min=63908 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96284 max_load_reduction_pct=0.592604 moved_to_replicas=295934 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[1366, 100574, 65564, 66281, 72793, 61377, 68312, 88021] avg=65536.0000 max=100574 max_rank=1 min=1366 min_rank=0 imbalance_ratio=1.534637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[1366, 100574, 65564, 66281, 72793, 61377, 68312, 88021] avg=65536.0000 max=100574 max_rank=1 min=1366 min_rank=0 imbalance_ratio=1.534637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63586 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=34381 max_load_reduction_pct=0.341848 moved_to_replicas=142652 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63586 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=34381 max_load_reduction_pct=0.341848 moved_to_replicas=142652 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63586 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=34381 max_load_reduction_pct=0.341848 moved_to_replicas=142652 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[1366, 100574, 65564, 66281, 72793, 61377, 68312, 88021] avg=65536.0000 max=100574 max_rank=1 min=1366 min_rank=0 imbalance_ratio=1.534637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63586 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=34381 max_load_reduction_pct=0.341848 moved_to_replicas=142652 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63586 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=34381 max_load_reduction_pct=0.341848 moved_to_replicas=142652 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[1366, 100574, 65564, 66281, 72793, 61377, 68312, 88021] avg=65536.0000 max=100574 max_rank=1 min=1366 min_rank=0 imbalance_ratio=1.534637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[1366, 100574, 65564, 66281, 72793, 61377, 68312, 88021] avg=65536.0000 max=100574 max_rank=1 min=1366 min_rank=0 imbalance_ratio=1.534637 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63586 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=34381 max_load_reduction_pct=0.341848 moved_to_replicas=142652 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[96452, 65885, 98055, 65492, 56054, 63684, 13126, 65540] avg=65536.0000 max=98055 max_rank=2 min=13126 min_rank=6 imbalance_ratio=1.496201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[96452, 65885, 98055, 65492, 56054, 63684, 13126, 65540] avg=65536.0000 max=98055 max_rank=2 min=13126 min_rank=6 imbalance_ratio=1.496201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[96452, 65885, 98055, 65492, 56054, 63684, 13126, 65540] avg=65536.0000 max=98055 max_rank=2 min=13126 min_rank=6 imbalance_ratio=1.496201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63586 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=34381 max_load_reduction_pct=0.341848 moved_to_replicas=142652 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[64177, 66192, 65564, 66192, 66192, 63586, 66192, 66193] avg=65536.0000 max=66193 max_rank=7 min=63586 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=34381 max_load_reduction_pct=0.341848 moved_to_replicas=142652 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[96452, 65885, 98055, 65492, 56054, 63684, 13126, 65540] avg=65536.0000 max=98055 max_rank=2 min=13126 min_rank=6 imbalance_ratio=1.496201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] avg=65536.0000 max=66192 max_rank=0 min=63684 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31863 max_load_reduction_pct=0.324950 moved_to_replicas=128105 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[96452, 65885, 98055, 65492, 56054, 63684, 13126, 65540] avg=65536.0000 max=98055 max_rank=2 min=13126 min_rank=6 imbalance_ratio=1.496201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] avg=65536.0000 max=66192 max_rank=0 min=63684 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31863 max_load_reduction_pct=0.324950 moved_to_replicas=128105 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] avg=65536.0000 max=66192 max_rank=0 min=63684 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31863 max_load_reduction_pct=0.324950 moved_to_replicas=128105 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[96452, 65885, 98055, 65492, 56054, 63684, 13126, 65540] avg=65536.0000 max=98055 max_rank=2 min=13126 min_rank=6 imbalance_ratio=1.496201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] avg=65536.0000 max=66192 max_rank=0 min=63684 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31863 max_load_reduction_pct=0.324950 moved_to_replicas=128105 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] avg=65536.0000 max=66192 max_rank=0 min=63684 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31863 max_load_reduction_pct=0.324950 moved_to_replicas=128105 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[96452, 65885, 98055, 65492, 56054, 63684, 13126, 65540] avg=65536.0000 max=98055 max_rank=2 min=13126 min_rank=6 imbalance_ratio=1.496201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[96452, 65885, 98055, 65492, 56054, 63684, 13126, 65540] avg=65536.0000 max=98055 max_rank=2 min=13126 min_rank=6 imbalance_ratio=1.496201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] avg=65536.0000 max=66192 max_rank=0 min=63684 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31863 max_load_reduction_pct=0.324950 moved_to_replicas=128105 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[55266, 65864, 97057, 32826, 131889, 19798, 1687, 119901] avg=65536.0000 max=131889 max_rank=4 min=1687 min_rank=6 imbalance_ratio=2.012466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[55266, 65864, 97057, 32826, 131889, 19798, 1687, 119901] avg=65536.0000 max=131889 max_rank=4 min=1687 min_rank=6 imbalance_ratio=2.012466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[55266, 65864, 97057, 32826, 131889, 19798, 1687, 119901] avg=65536.0000 max=131889 max_rank=4 min=1687 min_rank=6 imbalance_ratio=2.012466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] avg=65536.0000 max=66192 max_rank=0 min=63684 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31863 max_load_reduction_pct=0.324950 moved_to_replicas=128105 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[66192, 65885, 66192, 65492, 65111, 63684, 66192, 65540] avg=65536.0000 max=66192 max_rank=0 min=63684 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=31863 max_load_reduction_pct=0.324950 moved_to_replicas=128105 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[55266, 65864, 97057, 32826, 131889, 19798, 1687, 119901] avg=65536.0000 max=131889 max_rank=4 min=1687 min_rank=6 imbalance_ratio=2.012466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] avg=65536.0000 max=66192 max_rank=2 min=61730 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65697 max_load_reduction_pct=0.498123 moved_to_replicas=299684 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[55266, 65864, 97057, 32826, 131889, 19798, 1687, 119901] avg=65536.0000 max=131889 max_rank=4 min=1687 min_rank=6 imbalance_ratio=2.012466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] avg=65536.0000 max=66192 max_rank=2 min=61730 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65697 max_load_reduction_pct=0.498123 moved_to_replicas=299684 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] avg=65536.0000 max=66192 max_rank=2 min=61730 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65697 max_load_reduction_pct=0.498123 moved_to_replicas=299684 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[55266, 65864, 97057, 32826, 131889, 19798, 1687, 119901] avg=65536.0000 max=131889 max_rank=4 min=1687 min_rank=6 imbalance_ratio=2.012466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] avg=65536.0000 max=66192 max_rank=2 min=61730 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65697 max_load_reduction_pct=0.498123 moved_to_replicas=299684 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[55266, 65864, 97057, 32826, 131889, 19798, 1687, 119901] avg=65536.0000 max=131889 max_rank=4 min=1687 min_rank=6 imbalance_ratio=2.012466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] avg=65536.0000 max=66192 max_rank=2 min=61730 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65697 max_load_reduction_pct=0.498123 moved_to_replicas=299684 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[55266, 65864, 97057, 32826, 131889, 19798, 1687, 119901] avg=65536.0000 max=131889 max_rank=4 min=1687 min_rank=6 imbalance_ratio=2.012466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] avg=65536.0000 max=66192 max_rank=2 min=61730 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65697 max_load_reduction_pct=0.498123 moved_to_replicas=299684 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[35419, 107959, 33116, 127569, 27040, 65761, 97578, 29846] avg=65536.0000 max=127569 max_rank=3 min=27040 min_rank=4 imbalance_ratio=1.946548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[35419, 107959, 33116, 127569, 27040, 65761, 97578, 29846] avg=65536.0000 max=127569 max_rank=3 min=27040 min_rank=4 imbalance_ratio=1.946548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[35419, 107959, 33116, 127569, 27040, 65761, 97578, 29846] avg=65536.0000 max=127569 max_rank=3 min=27040 min_rank=4 imbalance_ratio=1.946548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] avg=65536.0000 max=66192 max_rank=2 min=61730 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65697 max_load_reduction_pct=0.498123 moved_to_replicas=299684 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[66191, 65864, 66192, 65736, 66192, 66192, 61730, 66191] avg=65536.0000 max=66192 max_rank=2 min=61730 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65697 max_load_reduction_pct=0.498123 moved_to_replicas=299684 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[35419, 107959, 33116, 127569, 27040, 65761, 97578, 29846] avg=65536.0000 max=127569 max_rank=3 min=27040 min_rank=4 imbalance_ratio=1.946548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=61714 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61376 max_load_reduction_pct=0.481120 moved_to_replicas=272118 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=61714 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61376 max_load_reduction_pct=0.481120 moved_to_replicas=272118 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[35419, 107959, 33116, 127569, 27040, 65761, 97578, 29846] avg=65536.0000 max=127569 max_rank=3 min=27040 min_rank=4 imbalance_ratio=1.946548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=61714 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61376 max_load_reduction_pct=0.481120 moved_to_replicas=272118 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[35419, 107959, 33116, 127569, 27040, 65761, 97578, 29846] avg=65536.0000 max=127569 max_rank=3 min=27040 min_rank=4 imbalance_ratio=1.946548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=61714 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61376 max_load_reduction_pct=0.481120 moved_to_replicas=272118 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[35419, 107959, 33116, 127569, 27040, 65761, 97578, 29846] avg=65536.0000 max=127569 max_rank=3 min=27040 min_rank=4 imbalance_ratio=1.946548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=61714 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61376 max_load_reduction_pct=0.481120 moved_to_replicas=272118 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[35419, 107959, 33116, 127569, 27040, 65761, 97578, 29846] avg=65536.0000 max=127569 max_rank=3 min=27040 min_rank=4 imbalance_ratio=1.946548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=61714 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61376 max_load_reduction_pct=0.481120 moved_to_replicas=272118 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[98694, 154, 65605, 56060, 97974, 129044, 45120, 31637] avg=65536.0000 max=129044 max_rank=5 min=154 min_rank=1 imbalance_ratio=1.969055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[98694, 154, 65605, 56060, 97974, 129044, 45120, 31637] avg=65536.0000 max=129044 max_rank=5 min=154 min_rank=1 imbalance_ratio=1.969055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[98694, 154, 65605, 56060, 97974, 129044, 45120, 31637] avg=65536.0000 max=129044 max_rank=5 min=154 min_rank=1 imbalance_ratio=1.969055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=61714 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61376 max_load_reduction_pct=0.481120 moved_to_replicas=272118 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[66193, 66191, 65853, 66192, 61714, 65761, 66192, 66192] avg=65536.0000 max=66193 max_rank=0 min=61714 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=61376 max_load_reduction_pct=0.481120 moved_to_replicas=272118 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[98694, 154, 65605, 56060, 97974, 129044, 45120, 31637] avg=65536.0000 max=129044 max_rank=5 min=154 min_rank=1 imbalance_ratio=1.969055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] avg=65536.0000 max=66192 max_rank=0 min=62332 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62852 max_load_reduction_pct=0.487059 moved_to_replicas=252296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] avg=65536.0000 max=66192 max_rank=0 min=62332 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62852 max_load_reduction_pct=0.487059 moved_to_replicas=252296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[98694, 154, 65605, 56060, 97974, 129044, 45120, 31637] avg=65536.0000 max=129044 max_rank=5 min=154 min_rank=1 imbalance_ratio=1.969055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] avg=65536.0000 max=66192 max_rank=0 min=62332 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62852 max_load_reduction_pct=0.487059 moved_to_replicas=252296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[98694, 154, 65605, 56060, 97974, 129044, 45120, 31637] avg=65536.0000 max=129044 max_rank=5 min=154 min_rank=1 imbalance_ratio=1.969055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] avg=65536.0000 max=66192 max_rank=0 min=62332 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62852 max_load_reduction_pct=0.487059 moved_to_replicas=252296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[98694, 154, 65605, 56060, 97974, 129044, 45120, 31637] avg=65536.0000 max=129044 max_rank=5 min=154 min_rank=1 imbalance_ratio=1.969055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] avg=65536.0000 max=66192 max_rank=0 min=62332 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62852 max_load_reduction_pct=0.487059 moved_to_replicas=252296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[98694, 154, 65605, 56060, 97974, 129044, 45120, 31637] avg=65536.0000 max=129044 max_rank=5 min=154 min_rank=1 imbalance_ratio=1.969055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[72052, 79752, 67756, 33005, 69792, 102434, 65610, 33887] avg=65536.0000 max=102434 max_rank=5 min=33005 min_rank=3 imbalance_ratio=1.563019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[72052, 79752, 67756, 33005, 69792, 102434, 65610, 33887] avg=65536.0000 max=102434 max_rank=5 min=33005 min_rank=3 imbalance_ratio=1.563019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] avg=65536.0000 max=66192 max_rank=0 min=62332 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62852 max_load_reduction_pct=0.487059 moved_to_replicas=252296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[72052, 79752, 67756, 33005, 69792, 102434, 65610, 33887] avg=65536.0000 max=102434 max_rank=5 min=33005 min_rank=3 imbalance_ratio=1.563019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] avg=65536.0000 max=66192 max_rank=0 min=62332 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62852 max_load_reduction_pct=0.487059 moved_to_replicas=252296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[72052, 79752, 67756, 33005, 69792, 102434, 65610, 33887] avg=65536.0000 max=102434 max_rank=5 min=33005 min_rank=3 imbalance_ratio=1.563019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] avg=65536.0000 max=66193 max_rank=1 min=61986 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=36241 max_load_reduction_pct=0.353799 moved_to_replicas=127828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] avg=65536.0000 max=66193 max_rank=1 min=61986 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=36241 max_load_reduction_pct=0.353799 moved_to_replicas=127828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[66192, 65391, 65605, 66192, 66192, 66192, 66192, 62332] avg=65536.0000 max=66192 max_rank=0 min=62332 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=62852 max_load_reduction_pct=0.487059 moved_to_replicas=252296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[72052, 79752, 67756, 33005, 69792, 102434, 65610, 33887] avg=65536.0000 max=102434 max_rank=5 min=33005 min_rank=3 imbalance_ratio=1.563019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] avg=65536.0000 max=66193 max_rank=1 min=61986 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=36241 max_load_reduction_pct=0.353799 moved_to_replicas=127828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[72052, 79752, 67756, 33005, 69792, 102434, 65610, 33887] avg=65536.0000 max=102434 max_rank=5 min=33005 min_rank=3 imbalance_ratio=1.563019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[72052, 79752, 67756, 33005, 69792, 102434, 65610, 33887] avg=65536.0000 max=102434 max_rank=5 min=33005 min_rank=3 imbalance_ratio=1.563019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] avg=65536.0000 max=66193 max_rank=1 min=61986 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=36241 max_load_reduction_pct=0.353799 moved_to_replicas=127828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] avg=65536.0000 max=66193 max_rank=1 min=61986 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=36241 max_load_reduction_pct=0.353799 moved_to_replicas=127828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[56153, 47090, 61172, 78435, 51518, 35156, 65530, 129234] avg=65536.0000 max=129234 max_rank=7 min=35156 min_rank=5 imbalance_ratio=1.971954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[56153, 47090, 61172, 78435, 51518, 35156, 65530, 129234] avg=65536.0000 max=129234 max_rank=7 min=35156 min_rank=5 imbalance_ratio=1.971954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] avg=65536.0000 max=66193 max_rank=1 min=61986 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=36241 max_load_reduction_pct=0.353799 moved_to_replicas=127828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[72052, 79752, 67756, 33005, 69792, 102434, 65610, 33887] avg=65536.0000 max=102434 max_rank=5 min=33005 min_rank=3 imbalance_ratio=1.563019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[56153, 47090, 61172, 78435, 51518, 35156, 65530, 129234] avg=65536.0000 max=129234 max_rank=7 min=35156 min_rank=5 imbalance_ratio=1.971954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] avg=65536.0000 max=66193 max_rank=1 min=61986 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=36241 max_load_reduction_pct=0.353799 moved_to_replicas=127828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] avg=65536.0000 max=66193 max_rank=1 min=63376 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63041 max_load_reduction_pct=0.487805 moved_to_replicas=147143 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[56153, 47090, 61172, 78435, 51518, 35156, 65530, 129234] avg=65536.0000 max=129234 max_rank=7 min=35156 min_rank=5 imbalance_ratio=1.971954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] avg=65536.0000 max=66193 max_rank=1 min=63376 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63041 max_load_reduction_pct=0.487805 moved_to_replicas=147143 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[66190, 66193, 66192, 65733, 66192, 66192, 65610, 61986] avg=65536.0000 max=66193 max_rank=1 min=61986 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=36241 max_load_reduction_pct=0.353799 moved_to_replicas=127828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[56153, 47090, 61172, 78435, 51518, 35156, 65530, 129234] avg=65536.0000 max=129234 max_rank=7 min=35156 min_rank=5 imbalance_ratio=1.971954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] avg=65536.0000 max=66193 max_rank=1 min=63376 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63041 max_load_reduction_pct=0.487805 moved_to_replicas=147143 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[56153, 47090, 61172, 78435, 51518, 35156, 65530, 129234] avg=65536.0000 max=129234 max_rank=7 min=35156 min_rank=5 imbalance_ratio=1.971954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] avg=65536.0000 max=66193 max_rank=1 min=63376 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63041 max_load_reduction_pct=0.487805 moved_to_replicas=147143 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[56153, 47090, 61172, 78435, 51518, 35156, 65530, 129234] avg=65536.0000 max=129234 max_rank=7 min=35156 min_rank=5 imbalance_ratio=1.971954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] avg=65536.0000 max=66193 max_rank=1 min=63376 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63041 max_load_reduction_pct=0.487805 moved_to_replicas=147143 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[98692, 68593, 32829, 2230, 33409, 93510, 161990, 33035] avg=65536.0000 max=161990 max_rank=6 min=2230 min_rank=3 imbalance_ratio=2.471771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[98692, 68593, 32829, 2230, 33409, 93510, 161990, 33035] avg=65536.0000 max=161990 max_rank=6 min=2230 min_rank=3 imbalance_ratio=2.471771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] avg=65536.0000 max=66193 max_rank=1 min=63376 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63041 max_load_reduction_pct=0.487805 moved_to_replicas=147143 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[56153, 47090, 61172, 78435, 51518, 35156, 65530, 129234] avg=65536.0000 max=129234 max_rank=7 min=35156 min_rank=5 imbalance_ratio=1.971954 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[98692, 68593, 32829, 2230, 33409, 93510, 161990, 33035] avg=65536.0000 max=161990 max_rank=6 min=2230 min_rank=3 imbalance_ratio=2.471771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] avg=65536.0000 max=66193 max_rank=1 min=63376 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63041 max_load_reduction_pct=0.487805 moved_to_replicas=147143 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] avg=65536.0000 max=66193 max_rank=5 min=63416 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=95797 max_load_reduction_pct=0.591376 moved_to_replicas=307324 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[98692, 68593, 32829, 2230, 33409, 93510, 161990, 33035] avg=65536.0000 max=161990 max_rank=6 min=2230 min_rank=3 imbalance_ratio=2.471771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] avg=65536.0000 max=66193 max_rank=5 min=63416 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=95797 max_load_reduction_pct=0.591376 moved_to_replicas=307324 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[98692, 68593, 32829, 2230, 33409, 93510, 161990, 33035] avg=65536.0000 max=161990 max_rank=6 min=2230 min_rank=3 imbalance_ratio=2.471771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[66192, 66193, 63376, 66192, 64421, 66192, 65530, 66192] avg=65536.0000 max=66193 max_rank=1 min=63376 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63041 max_load_reduction_pct=0.487805 moved_to_replicas=147143 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] avg=65536.0000 max=66193 max_rank=5 min=63416 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=95797 max_load_reduction_pct=0.591376 moved_to_replicas=307324 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[98692, 68593, 32829, 2230, 33409, 93510, 161990, 33035] avg=65536.0000 max=161990 max_rank=6 min=2230 min_rank=3 imbalance_ratio=2.471771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[98692, 68593, 32829, 2230, 33409, 93510, 161990, 33035] avg=65536.0000 max=161990 max_rank=6 min=2230 min_rank=3 imbalance_ratio=2.471771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] avg=65536.0000 max=66193 max_rank=5 min=63416 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=95797 max_load_reduction_pct=0.591376 moved_to_replicas=307324 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] avg=65536.0000 max=66193 max_rank=5 min=63416 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=95797 max_load_reduction_pct=0.591376 moved_to_replicas=307324 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[63364, 100548, 61808, 31762, 63703, 40823, 128534, 33746] avg=65536.0000 max=128534 max_rank=6 min=31762 min_rank=3 imbalance_ratio=1.961273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[63364, 100548, 61808, 31762, 63703, 40823, 128534, 33746] avg=65536.0000 max=128534 max_rank=6 min=31762 min_rank=3 imbalance_ratio=1.961273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] avg=65536.0000 max=66193 max_rank=5 min=63416 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=95797 max_load_reduction_pct=0.591376 moved_to_replicas=307324 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[98692, 68593, 32829, 2230, 33409, 93510, 161990, 33035] avg=65536.0000 max=161990 max_rank=6 min=2230 min_rank=3 imbalance_ratio=2.471771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] avg=65536.0000 max=66193 max_rank=5 min=63416 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=95797 max_load_reduction_pct=0.591376 moved_to_replicas=307324 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[63364, 100548, 61808, 31762, 63703, 40823, 128534, 33746] avg=65536.0000 max=128534 max_rank=6 min=31762 min_rank=3 imbalance_ratio=1.961273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63703 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62341 max_load_reduction_pct=0.485016 moved_to_replicas=194289 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63703 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62341 max_load_reduction_pct=0.485016 moved_to_replicas=194289 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[63364, 100548, 61808, 31762, 63703, 40823, 128534, 33746] avg=65536.0000 max=128534 max_rank=6 min=31762 min_rank=3 imbalance_ratio=1.961273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[63364, 100548, 61808, 31762, 63703, 40823, 128534, 33746] avg=65536.0000 max=128534 max_rank=6 min=31762 min_rank=3 imbalance_ratio=1.961273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[66192, 66192, 65532, 64662, 65909, 66193, 66192, 63416] avg=65536.0000 max=66193 max_rank=5 min=63416 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=95797 max_load_reduction_pct=0.591376 moved_to_replicas=307324 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63703 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62341 max_load_reduction_pct=0.485016 moved_to_replicas=194289 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[63364, 100548, 61808, 31762, 63703, 40823, 128534, 33746] avg=65536.0000 max=128534 max_rank=6 min=31762 min_rank=3 imbalance_ratio=1.961273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[63364, 100548, 61808, 31762, 63703, 40823, 128534, 33746] avg=65536.0000 max=128534 max_rank=6 min=31762 min_rank=3 imbalance_ratio=1.961273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63703 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62341 max_load_reduction_pct=0.485016 moved_to_replicas=194289 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[44565, 54548, 33046, 65443, 69314, 97414, 97201, 62757] avg=65536.0000 max=97414 max_rank=5 min=33046 min_rank=2 imbalance_ratio=1.486420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63703 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62341 max_load_reduction_pct=0.485016 moved_to_replicas=194289 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[44565, 54548, 33046, 65443, 69314, 97414, 97201, 62757] avg=65536.0000 max=97414 max_rank=5 min=33046 min_rank=2 imbalance_ratio=1.486420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63703 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62341 max_load_reduction_pct=0.485016 moved_to_replicas=194289 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[63364, 100548, 61808, 31762, 63703, 40823, 128534, 33746] avg=65536.0000 max=128534 max_rank=6 min=31762 min_rank=3 imbalance_ratio=1.961273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63703 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62341 max_load_reduction_pct=0.485016 moved_to_replicas=194289 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[44565, 54548, 33046, 65443, 69314, 97414, 97201, 62757] avg=65536.0000 max=97414 max_rank=5 min=33046 min_rank=2 imbalance_ratio=1.486420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] avg=65536.0000 max=66193 max_rank=0 min=63929 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=31221 max_load_reduction_pct=0.320498 moved_to_replicas=123957 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] avg=65536.0000 max=66193 max_rank=0 min=63929 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=31221 max_load_reduction_pct=0.320498 moved_to_replicas=123957 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[44565, 54548, 33046, 65443, 69314, 97414, 97201, 62757] avg=65536.0000 max=97414 max_rank=5 min=33046 min_rank=2 imbalance_ratio=1.486420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[44565, 54548, 33046, 65443, 69314, 97414, 97201, 62757] avg=65536.0000 max=97414 max_rank=5 min=33046 min_rank=2 imbalance_ratio=1.486420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[65124, 66192, 66193, 64501, 63703, 66191, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63703 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=62341 max_load_reduction_pct=0.485016 moved_to_replicas=194289 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] avg=65536.0000 max=66193 max_rank=0 min=63929 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=31221 max_load_reduction_pct=0.320498 moved_to_replicas=123957 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[44565, 54548, 33046, 65443, 69314, 97414, 97201, 62757] avg=65536.0000 max=97414 max_rank=5 min=33046 min_rank=2 imbalance_ratio=1.486420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[44565, 54548, 33046, 65443, 69314, 97414, 97201, 62757] avg=65536.0000 max=97414 max_rank=5 min=33046 min_rank=2 imbalance_ratio=1.486420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] avg=65536.0000 max=66193 max_rank=0 min=63929 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=31221 max_load_reduction_pct=0.320498 moved_to_replicas=123957 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[102085, 30877, 93218, 97927, 91906, 42082, 33822, 32371] avg=65536.0000 max=102085 max_rank=0 min=30877 min_rank=1 imbalance_ratio=1.557693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] avg=65536.0000 max=66193 max_rank=0 min=63929 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=31221 max_load_reduction_pct=0.320498 moved_to_replicas=123957 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[102085, 30877, 93218, 97927, 91906, 42082, 33822, 32371] avg=65536.0000 max=102085 max_rank=0 min=30877 min_rank=1 imbalance_ratio=1.557693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] avg=65536.0000 max=66193 max_rank=0 min=63929 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=31221 max_load_reduction_pct=0.320498 moved_to_replicas=123957 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] avg=65536.0000 max=66193 max_rank=0 min=63929 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=31221 max_load_reduction_pct=0.320498 moved_to_replicas=123957 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[44565, 54548, 33046, 65443, 69314, 97414, 97201, 62757] avg=65536.0000 max=97414 max_rank=5 min=33046 min_rank=2 imbalance_ratio=1.486420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[102085, 30877, 93218, 97927, 91906, 42082, 33822, 32371] avg=65536.0000 max=102085 max_rank=0 min=30877 min_rank=1 imbalance_ratio=1.557693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] avg=65536.0000 max=66193 max_rank=4 min=63504 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35892 max_load_reduction_pct=0.351589 moved_to_replicas=236579 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] avg=65536.0000 max=66193 max_rank=4 min=63504 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35892 max_load_reduction_pct=0.351589 moved_to_replicas=236579 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[102085, 30877, 93218, 97927, 91906, 42082, 33822, 32371] avg=65536.0000 max=102085 max_rank=0 min=30877 min_rank=1 imbalance_ratio=1.557693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[66193, 63929, 64268, 65443, 66192, 66192, 66192, 65879] avg=65536.0000 max=66193 max_rank=0 min=63929 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=31221 max_load_reduction_pct=0.320498 moved_to_replicas=123957 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[102085, 30877, 93218, 97927, 91906, 42082, 33822, 32371] avg=65536.0000 max=102085 max_rank=0 min=30877 min_rank=1 imbalance_ratio=1.557693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] avg=65536.0000 max=66193 max_rank=4 min=63504 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35892 max_load_reduction_pct=0.351589 moved_to_replicas=236579 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[102085, 30877, 93218, 97927, 91906, 42082, 33822, 32371] avg=65536.0000 max=102085 max_rank=0 min=30877 min_rank=1 imbalance_ratio=1.557693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[102085, 30877, 93218, 97927, 91906, 42082, 33822, 32371] avg=65536.0000 max=102085 max_rank=0 min=30877 min_rank=1 imbalance_ratio=1.557693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] avg=65536.0000 max=66193 max_rank=4 min=63504 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35892 max_load_reduction_pct=0.351589 moved_to_replicas=236579 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[65573, 131580, 13142, 42549, 32638, 80586, 63221, 94999] avg=65536.0000 max=131580 max_rank=1 min=13142 min_rank=2 imbalance_ratio=2.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[65573, 131580, 13142, 42549, 32638, 80586, 63221, 94999] avg=65536.0000 max=131580 max_rank=1 min=13142 min_rank=2 imbalance_ratio=2.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] avg=65536.0000 max=66193 max_rank=4 min=63504 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35892 max_load_reduction_pct=0.351589 moved_to_replicas=236579 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] avg=65536.0000 max=66193 max_rank=4 min=63504 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35892 max_load_reduction_pct=0.351589 moved_to_replicas=236579 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] avg=65536.0000 max=66193 max_rank=4 min=63504 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35892 max_load_reduction_pct=0.351589 moved_to_replicas=236579 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[102085, 30877, 93218, 97927, 91906, 42082, 33822, 32371] avg=65536.0000 max=102085 max_rank=0 min=30877 min_rank=1 imbalance_ratio=1.557693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[65573, 131580, 13142, 42549, 32638, 80586, 63221, 94999] avg=65536.0000 max=131580 max_rank=1 min=13142 min_rank=2 imbalance_ratio=2.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] avg=65536.0000 max=66192 max_rank=1 min=63221 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65388 max_load_reduction_pct=0.496945 moved_to_replicas=205605 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] avg=65536.0000 max=66192 max_rank=1 min=63221 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65388 max_load_reduction_pct=0.496945 moved_to_replicas=205605 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[65573, 131580, 13142, 42549, 32638, 80586, 63221, 94999] avg=65536.0000 max=131580 max_rank=1 min=13142 min_rank=2 imbalance_ratio=2.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[65573, 131580, 13142, 42549, 32638, 80586, 63221, 94999] avg=65536.0000 max=131580 max_rank=1 min=13142 min_rank=2 imbalance_ratio=2.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[66192, 63504, 66192, 66192, 66193, 66191, 65557, 64267] avg=65536.0000 max=66193 max_rank=4 min=63504 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35892 max_load_reduction_pct=0.351589 moved_to_replicas=236579 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] avg=65536.0000 max=66192 max_rank=1 min=63221 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65388 max_load_reduction_pct=0.496945 moved_to_replicas=205605 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[65573, 131580, 13142, 42549, 32638, 80586, 63221, 94999] avg=65536.0000 max=131580 max_rank=1 min=13142 min_rank=2 imbalance_ratio=2.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[65573, 131580, 13142, 42549, 32638, 80586, 63221, 94999] avg=65536.0000 max=131580 max_rank=1 min=13142 min_rank=2 imbalance_ratio=2.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32880, 52831, 50842, 66579, 131046, 32876, 33261, 123973] avg=65536.0000 max=131046 max_rank=4 min=32876 min_rank=5 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32880, 52831, 50842, 66579, 131046, 32876, 33261, 123973] avg=65536.0000 max=131046 max_rank=4 min=32876 min_rank=5 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] avg=65536.0000 max=66192 max_rank=1 min=63221 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65388 max_load_reduction_pct=0.496945 moved_to_replicas=205605 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] avg=65536.0000 max=66192 max_rank=1 min=63221 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65388 max_load_reduction_pct=0.496945 moved_to_replicas=205605 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] avg=65536.0000 max=66192 max_rank=1 min=63221 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65388 max_load_reduction_pct=0.496945 moved_to_replicas=205605 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] avg=65536.0000 max=66192 max_rank=1 min=63221 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65388 max_load_reduction_pct=0.496945 moved_to_replicas=205605 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32880, 52831, 50842, 66579, 131046, 32876, 33261, 123973] avg=65536.0000 max=131046 max_rank=4 min=32876 min_rank=5 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[65573, 131580, 13142, 42549, 32638, 80586, 63221, 94999] avg=65536.0000 max=131580 max_rank=1 min=13142 min_rank=2 imbalance_ratio=2.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] avg=65536.0000 max=66192 max_rank=2 min=63865 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=239121 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] avg=65536.0000 max=66192 max_rank=2 min=63865 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=239121 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32880, 52831, 50842, 66579, 131046, 32876, 33261, 123973] avg=65536.0000 max=131046 max_rank=4 min=32876 min_rank=5 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32880, 52831, 50842, 66579, 131046, 32876, 33261, 123973] avg=65536.0000 max=131046 max_rank=4 min=32876 min_rank=5 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] avg=65536.0000 max=66192 max_rank=2 min=63865 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=239121 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32880, 52831, 50842, 66579, 131046, 32876, 33261, 123973] avg=65536.0000 max=131046 max_rank=4 min=32876 min_rank=5 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[65573, 66192, 65426, 66191, 65301, 66192, 63221, 66192] avg=65536.0000 max=66192 max_rank=1 min=63221 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65388 max_load_reduction_pct=0.496945 moved_to_replicas=205605 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32880, 52831, 50842, 66579, 131046, 32876, 33261, 123973] avg=65536.0000 max=131046 max_rank=4 min=32876 min_rank=5 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[59395, 81019, 60995, 47651, 96102, 57025, 58101, 64000] avg=65536.0000 max=96102 max_rank=4 min=47651 min_rank=3 imbalance_ratio=1.466400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[59395, 81019, 60995, 47651, 96102, 57025, 58101, 64000] avg=65536.0000 max=96102 max_rank=4 min=47651 min_rank=3 imbalance_ratio=1.466400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] avg=65536.0000 max=66192 max_rank=2 min=63865 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=239121 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] avg=65536.0000 max=66192 max_rank=2 min=63865 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=239121 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] avg=65536.0000 max=66192 max_rank=2 min=63865 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=239121 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] avg=65536.0000 max=66192 max_rank=2 min=63865 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=239121 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[59395, 81019, 60995, 47651, 96102, 57025, 58101, 64000] avg=65536.0000 max=96102 max_rank=4 min=47651 min_rank=3 imbalance_ratio=1.466400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32880, 52831, 50842, 66579, 131046, 32876, 33261, 123973] avg=65536.0000 max=131046 max_rank=4 min=32876 min_rank=5 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] avg=65536.0000 max=66193 max_rank=6 min=63136 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29909 max_load_reduction_pct=0.311221 moved_to_replicas=87543 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] avg=65536.0000 max=66193 max_rank=6 min=63136 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29909 max_load_reduction_pct=0.311221 moved_to_replicas=87543 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[59395, 81019, 60995, 47651, 96102, 57025, 58101, 64000] avg=65536.0000 max=96102 max_rank=4 min=47651 min_rank=3 imbalance_ratio=1.466400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] avg=65536.0000 max=66193 max_rank=6 min=63136 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29909 max_load_reduction_pct=0.311221 moved_to_replicas=87543 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[59395, 81019, 60995, 47651, 96102, 57025, 58101, 64000] avg=65536.0000 max=96102 max_rank=4 min=47651 min_rank=3 imbalance_ratio=1.466400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[59395, 81019, 60995, 47651, 96102, 57025, 58101, 64000] avg=65536.0000 max=96102 max_rank=4 min=47651 min_rank=3 imbalance_ratio=1.466400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[59395, 81019, 60995, 47651, 96102, 57025, 58101, 64000] avg=65536.0000 max=96102 max_rank=4 min=47651 min_rank=3 imbalance_ratio=1.466400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[65006, 63865, 66192, 66192, 66192, 65604, 65046, 66191] avg=65536.0000 max=66192 max_rank=2 min=63865 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64854 max_load_reduction_pct=0.494895 moved_to_replicas=239121 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] avg=65536.0000 max=66193 max_rank=6 min=63136 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29909 max_load_reduction_pct=0.311221 moved_to_replicas=87543 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] avg=65536.0000 max=66193 max_rank=6 min=63136 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29909 max_load_reduction_pct=0.311221 moved_to_replicas=87543 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] avg=65536.0000 max=66193 max_rank=6 min=63136 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29909 max_load_reduction_pct=0.311221 moved_to_replicas=87543 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] avg=65536.0000 max=66193 max_rank=6 min=63136 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29909 max_load_reduction_pct=0.311221 moved_to_replicas=87543 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[59395, 81019, 60995, 47651, 96102, 57025, 58101, 64000] avg=65536.0000 max=96102 max_rank=4 min=47651 min_rank=3 imbalance_ratio=1.466400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[66192, 66191, 63136, 66192, 66192, 66192, 66193, 64000] avg=65536.0000 max=66193 max_rank=6 min=63136 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=29909 max_load_reduction_pct=0.311221 moved_to_replicas=87543 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=2103578 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=495488 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=2102407 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=496768 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=2095444 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=503808 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=2099056 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=499328 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=2079925 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=519552 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=2097334 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=501888 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=2104713 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=494592 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=2094759 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=504064 peak_demand=66193 required_capacity=1.010025 + [2026-08-30 05:14:10] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 550.7 | throughput per GPU (TFLOP/s/GPU): 605.4 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.316051E+01 | loss scale: 1.0 | grad norm: 38.033 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=81920 expected_recv=128095 expected_overflow=0 exact_recv_per_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=81920 expected_recv=129740 expected_overflow=0 exact_recv_per_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=81920 expected_recv=128822 expected_overflow=0 exact_recv_per_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=81920 expected_recv=128095 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=81920 expected_recv=129740 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=81920 expected_recv=128822 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=81920 expected_recv=130697 expected_overflow=0 exact_recv_per_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=81920 expected_recv=132205 expected_overflow=0 exact_recv_per_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=81920 expected_recv=130697 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131110 expected_overflow=0 exact_recv_per_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=81920 expected_recv=132205 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=81920 expected_recv=129937 expected_overflow=0 exact_recv_per_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=81920 expected_recv=127475 expected_overflow=0 exact_recv_per_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131110 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=81920 expected_recv=129937 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=81920 expected_recv=127475 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=129408 slack_slots=34432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=81920 expected_recv=127924 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=81920 expected_recv=128556 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131146 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131416 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=81920 expected_recv=128556 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=81920 expected_recv=127924 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131146 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131416 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=129408 slack_slots=34432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=81920 expected_recv=130927 expected_overflow=0 exact_recv_per_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131148 expected_overflow=0 exact_recv_per_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=81920 expected_recv=130927 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=81920 expected_recv=129887 expected_overflow=0 exact_recv_per_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131852 expected_overflow=0 exact_recv_per_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131017 expected_overflow=0 exact_recv_per_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131400 expected_overflow=0 exact_recv_per_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131148 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=81920 expected_recv=129887 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131852 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131017 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=81920 expected_recv=129961 expected_overflow=0 exact_recv_per_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131400 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130430 expected_overflow=0 exact_recv_per_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=81920 expected_recv=129961 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130430 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=81920 expected_recv=129745 expected_overflow=0 exact_recv_per_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130443 expected_overflow=0 exact_recv_per_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=81920 expected_recv=129874 expected_overflow=0 exact_recv_per_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=81920 expected_recv=129745 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130932 expected_overflow=0 exact_recv_per_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130443 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=81920 expected_recv=129874 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130932 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131065 expected_overflow=0 exact_recv_per_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131065 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=81920 expected_recv=127981 expected_overflow=0 exact_recv_per_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131577 expected_overflow=0 exact_recv_per_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=81920 expected_recv=127981 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=81920 expected_recv=130256 expected_overflow=0 exact_recv_per_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131217 expected_overflow=0 exact_recv_per_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131577 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131712 expected_overflow=0 exact_recv_per_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=81920 expected_recv=130256 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=81920 expected_recv=130203 expected_overflow=0 exact_recv_per_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131217 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131712 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=81920 expected_recv=128161 expected_overflow=0 exact_recv_per_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=81920 expected_recv=130203 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=81920 expected_recv=128161 actual_valid_recv=129408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=129408 slack_slots=34432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=81920 expected_recv=129645 expected_overflow=0 exact_recv_per_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=81920 expected_recv=132373 expected_overflow=0 exact_recv_per_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=81920 expected_recv=129645 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=81920 expected_recv=132373 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131042 expected_overflow=0 exact_recv_per_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131042 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=81920 expected_recv=130873 expected_overflow=0 exact_recv_per_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=81920 expected_recv=130873 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=81920 expected_recv=129769 expected_overflow=0 exact_recv_per_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=81920 expected_recv=131143 expected_overflow=0 exact_recv_per_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=81920 expected_recv=129930 expected_overflow=0 exact_recv_per_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=81920 expected_recv=129769 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=81920 expected_recv=131143 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=81920 expected_recv=129930 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=81920 expected_recv=129709 expected_overflow=0 exact_recv_per_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=81920 expected_recv=129709 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=81920 expected_recv=129743 expected_overflow=0 exact_recv_per_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=81920 expected_recv=129743 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=81920 expected_recv=132079 expected_overflow=0 exact_recv_per_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=81920 expected_recv=129717 expected_overflow=0 exact_recv_per_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=81920 expected_recv=132079 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=81920 expected_recv=129717 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=81920 expected_recv=130858 expected_overflow=0 exact_recv_per_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=81920 expected_recv=130858 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=81920 expected_recv=132064 expected_overflow=0 exact_recv_per_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=81920 expected_recv=129026 expected_overflow=0 exact_recv_per_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131351 expected_overflow=0 exact_recv_per_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=81920 expected_recv=132064 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=81920 expected_recv=129026 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131351 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131180 expected_overflow=0 exact_recv_per_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=81920 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=81920 expected_recv=130577 expected_overflow=0 exact_recv_per_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131180 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=81920 expected_recv=132387 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=81920 expected_recv=130577 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=81920 expected_recv=126249 expected_overflow=0 exact_recv_per_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=81920 expected_recv=130396 expected_overflow=0 exact_recv_per_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=81920 expected_recv=126249 actual_valid_recv=127744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=81920 expected_recv=130396 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=127744 slack_slots=36096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=81920 expected_recv=129795 expected_overflow=0 exact_recv_per_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=81920 expected_recv=129158 expected_overflow=0 exact_recv_per_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=81920 expected_recv=129795 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=81920 expected_recv=129158 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131761 expected_overflow=0 exact_recv_per_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=81920 expected_recv=123685 expected_overflow=0 exact_recv_per_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131761 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=81920 expected_recv=130965 expected_overflow=0 exact_recv_per_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131733 expected_overflow=0 exact_recv_per_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=81920 expected_recv=123685 actual_valid_recv=125312 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=81920 expected_recv=130965 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=125312 slack_slots=38528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131733 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132387 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=81920 expected_recv=130583 expected_overflow=0 exact_recv_per_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=81920 expected_recv=132164 expected_overflow=0 exact_recv_per_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=81920 expected_recv=130583 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=81920 expected_recv=132164 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131422 expected_overflow=0 exact_recv_per_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131422 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=81920 expected_recv=124367 expected_overflow=0 exact_recv_per_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=81920 expected_recv=129981 expected_overflow=0 exact_recv_per_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=81920 expected_recv=124367 actual_valid_recv=125440 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=81920 expected_recv=129981 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=81920 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131084 expected_overflow=0 exact_recv_per_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=125440 slack_slots=38400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=81920 expected_recv=132387 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131084 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=81920 expected_recv=127440 expected_overflow=0 exact_recv_per_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=81920 expected_recv=129705 expected_overflow=0 exact_recv_per_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=81920 expected_recv=127440 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=81920 expected_recv=129705 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=81920 expected_recv=129579 expected_overflow=0 exact_recv_per_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=128512 slack_slots=35328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=81920 expected_recv=129579 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=81920 expected_recv=129774 expected_overflow=0 exact_recv_per_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131051 expected_overflow=0 exact_recv_per_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=81920 expected_recv=130552 expected_overflow=0 exact_recv_per_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=81920 expected_recv=129774 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131051 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=81920 expected_recv=130552 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=81920 expected_recv=128757 expected_overflow=0 exact_recv_per_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131265 expected_overflow=0 exact_recv_per_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=81920 expected_recv=128757 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131265 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=81920 expected_recv=128968 expected_overflow=0 exact_recv_per_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131117 expected_overflow=0 exact_recv_per_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=81920 expected_recv=130839 expected_overflow=0 exact_recv_per_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=81920 expected_recv=128968 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131117 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=81920 expected_recv=130839 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=129792 slack_slots=34048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131092 expected_overflow=0 exact_recv_per_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[33317, 51818, 101750, 52098, 74413, 85374, 44327, 81191] avg=65536.0000 max=101750 max_rank=2 min=33317 min_rank=0 imbalance_ratio=1.552582 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131092 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131768 expected_overflow=0 exact_recv_per_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[33317, 51818, 101750, 52098, 74413, 85374, 44327, 81191] avg=65536.0000 max=101750 max_rank=2 min=33317 min_rank=0 imbalance_ratio=1.552582 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[33317, 51818, 101750, 52098, 74413, 85374, 44327, 81191] avg=65536.0000 max=101750 max_rank=2 min=33317 min_rank=0 imbalance_ratio=1.552582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[33317, 51818, 101750, 52098, 74413, 85374, 44327, 81191] avg=65536.0000 max=101750 max_rank=2 min=33317 min_rank=0 imbalance_ratio=1.552582 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131768 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[33317, 51818, 101750, 52098, 74413, 85374, 44327, 81191] avg=65536.0000 max=101750 max_rank=2 min=33317 min_rank=0 imbalance_ratio=1.552582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=63548 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35557 max_load_reduction_pct=0.349455 moved_to_replicas=158764 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=63548 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35557 max_load_reduction_pct=0.349455 moved_to_replicas=158764 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=63548 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35557 max_load_reduction_pct=0.349455 moved_to_replicas=158764 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=63548 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35557 max_load_reduction_pct=0.349455 moved_to_replicas=158764 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=63548 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35557 max_load_reduction_pct=0.349455 moved_to_replicas=158764 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[33317, 51818, 101750, 52098, 74413, 85374, 44327, 81191] avg=65536.0000 max=101750 max_rank=2 min=33317 min_rank=0 imbalance_ratio=1.552582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[33317, 51818, 101750, 52098, 74413, 85374, 44327, 81191] avg=65536.0000 max=101750 max_rank=2 min=33317 min_rank=0 imbalance_ratio=1.552582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33090, 131026, 66013, 35754, 97317, 65480, 32706, 62902] avg=65536.0000 max=131026 max_rank=1 min=32706 min_rank=6 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[33317, 51818, 101750, 52098, 74413, 85374, 44327, 81191] avg=65536.0000 max=101750 max_rank=2 min=33317 min_rank=0 imbalance_ratio=1.552582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33090, 131026, 66013, 35754, 97317, 65480, 32706, 62902] avg=65536.0000 max=131026 max_rank=1 min=32706 min_rank=6 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33090, 131026, 66013, 35754, 97317, 65480, 32706, 62902] avg=65536.0000 max=131026 max_rank=1 min=32706 min_rank=6 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33090, 131026, 66013, 35754, 97317, 65480, 32706, 62902] avg=65536.0000 max=131026 max_rank=1 min=32706 min_rank=6 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] avg=65536.0000 max=66192 max_rank=1 min=63589 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=192032 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=63548 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35557 max_load_reduction_pct=0.349455 moved_to_replicas=158764 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=63548 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35557 max_load_reduction_pct=0.349455 moved_to_replicas=158764 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33090, 131026, 66013, 35754, 97317, 65480, 32706, 62902] avg=65536.0000 max=131026 max_rank=1 min=32706 min_rank=6 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[63588, 63548, 66191, 66191, 66192, 66192, 66193, 66193] avg=65536.0000 max=66193 max_rank=6 min=63548 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=35557 max_load_reduction_pct=0.349455 moved_to_replicas=158764 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] avg=65536.0000 max=66192 max_rank=1 min=63589 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=192032 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] avg=65536.0000 max=66192 max_rank=1 min=63589 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=192032 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] avg=65536.0000 max=66192 max_rank=1 min=63589 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=192032 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] avg=65536.0000 max=66192 max_rank=1 min=63589 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=192032 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[95532, 98220, 90411, 34646, 65869, 7232, 66755, 65623] avg=65536.0000 max=98220 max_rank=1 min=7232 min_rank=5 imbalance_ratio=1.498718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33090, 131026, 66013, 35754, 97317, 65480, 32706, 62902] avg=65536.0000 max=131026 max_rank=1 min=32706 min_rank=6 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33090, 131026, 66013, 35754, 97317, 65480, 32706, 62902] avg=65536.0000 max=131026 max_rank=1 min=32706 min_rank=6 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33090, 131026, 66013, 35754, 97317, 65480, 32706, 62902] avg=65536.0000 max=131026 max_rank=1 min=32706 min_rank=6 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[95532, 98220, 90411, 34646, 65869, 7232, 66755, 65623] avg=65536.0000 max=98220 max_rank=1 min=7232 min_rank=5 imbalance_ratio=1.498718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[95532, 98220, 90411, 34646, 65869, 7232, 66755, 65623] avg=65536.0000 max=98220 max_rank=1 min=7232 min_rank=5 imbalance_ratio=1.498718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[95532, 98220, 90411, 34646, 65869, 7232, 66755, 65623] avg=65536.0000 max=98220 max_rank=1 min=7232 min_rank=5 imbalance_ratio=1.498718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] avg=65536.0000 max=66192 max_rank=0 min=63986 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32028 max_load_reduction_pct=0.326084 moved_to_replicas=169695 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] avg=65536.0000 max=66192 max_rank=1 min=63589 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=192032 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] avg=65536.0000 max=66192 max_rank=1 min=63589 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=192032 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[95532, 98220, 90411, 34646, 65869, 7232, 66755, 65623] avg=65536.0000 max=98220 max_rank=1 min=7232 min_rank=5 imbalance_ratio=1.498718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[65181, 66192, 66013, 66192, 66192, 65480, 65449, 63589] avg=65536.0000 max=66192 max_rank=1 min=63589 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64834 max_load_reduction_pct=0.494818 moved_to_replicas=192032 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] avg=65536.0000 max=66192 max_rank=0 min=63986 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32028 max_load_reduction_pct=0.326084 moved_to_replicas=169695 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] avg=65536.0000 max=66192 max_rank=0 min=63986 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32028 max_load_reduction_pct=0.326084 moved_to_replicas=169695 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] avg=65536.0000 max=66192 max_rank=0 min=63986 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32028 max_load_reduction_pct=0.326084 moved_to_replicas=169695 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] avg=65536.0000 max=66192 max_rank=0 min=63986 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32028 max_load_reduction_pct=0.326084 moved_to_replicas=169695 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[98180, 65519, 65625, 65457, 64351, 32918, 66657, 65581] avg=65536.0000 max=98180 max_rank=0 min=32918 min_rank=5 imbalance_ratio=1.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[95532, 98220, 90411, 34646, 65869, 7232, 66755, 65623] avg=65536.0000 max=98220 max_rank=1 min=7232 min_rank=5 imbalance_ratio=1.498718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[95532, 98220, 90411, 34646, 65869, 7232, 66755, 65623] avg=65536.0000 max=98220 max_rank=1 min=7232 min_rank=5 imbalance_ratio=1.498718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[95532, 98220, 90411, 34646, 65869, 7232, 66755, 65623] avg=65536.0000 max=98220 max_rank=1 min=7232 min_rank=5 imbalance_ratio=1.498718 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[98180, 65519, 65625, 65457, 64351, 32918, 66657, 65581] avg=65536.0000 max=98180 max_rank=0 min=32918 min_rank=5 imbalance_ratio=1.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[98180, 65519, 65625, 65457, 64351, 32918, 66657, 65581] avg=65536.0000 max=98180 max_rank=0 min=32918 min_rank=5 imbalance_ratio=1.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[98180, 65519, 65625, 65457, 64351, 32918, 66657, 65581] avg=65536.0000 max=98180 max_rank=0 min=32918 min_rank=5 imbalance_ratio=1.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] avg=65536.0000 max=66192 max_rank=0 min=64816 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31988 max_load_reduction_pct=0.325810 moved_to_replicas=64459 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] avg=65536.0000 max=66192 max_rank=0 min=63986 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32028 max_load_reduction_pct=0.326084 moved_to_replicas=169695 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[98180, 65519, 65625, 65457, 64351, 32918, 66657, 65581] avg=65536.0000 max=98180 max_rank=0 min=32918 min_rank=5 imbalance_ratio=1.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] avg=65536.0000 max=66192 max_rank=0 min=63986 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32028 max_load_reduction_pct=0.326084 moved_to_replicas=169695 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[66192, 66192, 66191, 63986, 65869, 64043, 66192, 65623] avg=65536.0000 max=66192 max_rank=0 min=63986 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32028 max_load_reduction_pct=0.326084 moved_to_replicas=169695 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] avg=65536.0000 max=66192 max_rank=0 min=64816 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31988 max_load_reduction_pct=0.325810 moved_to_replicas=64459 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] avg=65536.0000 max=66192 max_rank=0 min=64816 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31988 max_load_reduction_pct=0.325810 moved_to_replicas=64459 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] avg=65536.0000 max=66192 max_rank=0 min=64816 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31988 max_load_reduction_pct=0.325810 moved_to_replicas=64459 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32855, 98004, 64676, 65456, 98907, 32999, 33123, 98268] avg=65536.0000 max=98907 max_rank=4 min=32855 min_rank=0 imbalance_ratio=1.509201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] avg=65536.0000 max=66192 max_rank=0 min=64816 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31988 max_load_reduction_pct=0.325810 moved_to_replicas=64459 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[98180, 65519, 65625, 65457, 64351, 32918, 66657, 65581] avg=65536.0000 max=98180 max_rank=0 min=32918 min_rank=5 imbalance_ratio=1.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[98180, 65519, 65625, 65457, 64351, 32918, 66657, 65581] avg=65536.0000 max=98180 max_rank=0 min=32918 min_rank=5 imbalance_ratio=1.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[98180, 65519, 65625, 65457, 64351, 32918, 66657, 65581] avg=65536.0000 max=98180 max_rank=0 min=32918 min_rank=5 imbalance_ratio=1.498108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32855, 98004, 64676, 65456, 98907, 32999, 33123, 98268] avg=65536.0000 max=98907 max_rank=4 min=32855 min_rank=0 imbalance_ratio=1.509201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32855, 98004, 64676, 65456, 98907, 32999, 33123, 98268] avg=65536.0000 max=98907 max_rank=4 min=32855 min_rank=0 imbalance_ratio=1.509201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] avg=65536.0000 max=66192 max_rank=1 min=64676 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.330765 moved_to_replicas=193249 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32855, 98004, 64676, 65456, 98907, 32999, 33123, 98268] avg=65536.0000 max=98907 max_rank=4 min=32855 min_rank=0 imbalance_ratio=1.509201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] avg=65536.0000 max=66192 max_rank=0 min=64816 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31988 max_load_reduction_pct=0.325810 moved_to_replicas=64459 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] avg=65536.0000 max=66192 max_rank=0 min=64816 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31988 max_load_reduction_pct=0.325810 moved_to_replicas=64459 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32855, 98004, 64676, 65456, 98907, 32999, 33123, 98268] avg=65536.0000 max=98907 max_rank=4 min=32855 min_rank=0 imbalance_ratio=1.509201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[66192, 65519, 65625, 65457, 64816, 64906, 66192, 65581] avg=65536.0000 max=66192 max_rank=0 min=64816 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=31988 max_load_reduction_pct=0.325810 moved_to_replicas=64459 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] avg=65536.0000 max=66192 max_rank=1 min=64676 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.330765 moved_to_replicas=193249 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] avg=65536.0000 max=66192 max_rank=1 min=64676 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.330765 moved_to_replicas=193249 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] avg=65536.0000 max=66192 max_rank=1 min=64676 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.330765 moved_to_replicas=193249 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32774, 153464, 38036, 65500, 65525, 65863, 37951, 65175] avg=65536.0000 max=153464 max_rank=1 min=32774 min_rank=0 imbalance_ratio=2.341675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] avg=65536.0000 max=66192 max_rank=1 min=64676 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.330765 moved_to_replicas=193249 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32855, 98004, 64676, 65456, 98907, 32999, 33123, 98268] avg=65536.0000 max=98907 max_rank=4 min=32855 min_rank=0 imbalance_ratio=1.509201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32855, 98004, 64676, 65456, 98907, 32999, 33123, 98268] avg=65536.0000 max=98907 max_rank=4 min=32855 min_rank=0 imbalance_ratio=1.509201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32774, 153464, 38036, 65500, 65525, 65863, 37951, 65175] avg=65536.0000 max=153464 max_rank=1 min=32774 min_rank=0 imbalance_ratio=2.341675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32855, 98004, 64676, 65456, 98907, 32999, 33123, 98268] avg=65536.0000 max=98907 max_rank=4 min=32855 min_rank=0 imbalance_ratio=1.509201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32774, 153464, 38036, 65500, 65525, 65863, 37951, 65175] avg=65536.0000 max=153464 max_rank=1 min=32774 min_rank=0 imbalance_ratio=2.341675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] avg=65536.0000 max=66193 max_rank=6 min=64301 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=87271 max_load_reduction_pct=0.568674 moved_to_replicas=169962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32774, 153464, 38036, 65500, 65525, 65863, 37951, 65175] avg=65536.0000 max=153464 max_rank=1 min=32774 min_rank=0 imbalance_ratio=2.341675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] avg=65536.0000 max=66192 max_rank=1 min=64676 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.330765 moved_to_replicas=193249 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32774, 153464, 38036, 65500, 65525, 65863, 37951, 65175] avg=65536.0000 max=153464 max_rank=1 min=32774 min_rank=0 imbalance_ratio=2.341675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] avg=65536.0000 max=66192 max_rank=1 min=64676 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.330765 moved_to_replicas=193249 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] avg=65536.0000 max=66193 max_rank=6 min=64301 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=87271 max_load_reduction_pct=0.568674 moved_to_replicas=169962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] avg=65536.0000 max=66193 max_rank=6 min=64301 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=87271 max_load_reduction_pct=0.568674 moved_to_replicas=169962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[65570, 66192, 64676, 65456, 66192, 65075, 64935, 66192] avg=65536.0000 max=66192 max_rank=1 min=64676 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32715 max_load_reduction_pct=0.330765 moved_to_replicas=193249 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] avg=65536.0000 max=66193 max_rank=6 min=64301 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=87271 max_load_reduction_pct=0.568674 moved_to_replicas=169962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[62892, 26742, 32772, 115669, 65525, 86348, 67029, 67311] avg=65536.0000 max=115669 max_rank=3 min=26742 min_rank=1 imbalance_ratio=1.764969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] avg=65536.0000 max=66193 max_rank=6 min=64301 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=87271 max_load_reduction_pct=0.568674 moved_to_replicas=169962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32774, 153464, 38036, 65500, 65525, 65863, 37951, 65175] avg=65536.0000 max=153464 max_rank=1 min=32774 min_rank=0 imbalance_ratio=2.341675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32774, 153464, 38036, 65500, 65525, 65863, 37951, 65175] avg=65536.0000 max=153464 max_rank=1 min=32774 min_rank=0 imbalance_ratio=2.341675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[62892, 26742, 32772, 115669, 65525, 86348, 67029, 67311] avg=65536.0000 max=115669 max_rank=3 min=26742 min_rank=1 imbalance_ratio=1.764969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] avg=65536.0000 max=66194 max_rank=5 min=63792 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49475 max_load_reduction_pct=0.427729 moved_to_replicas=138511 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[62892, 26742, 32772, 115669, 65525, 86348, 67029, 67311] avg=65536.0000 max=115669 max_rank=3 min=26742 min_rank=1 imbalance_ratio=1.764969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32774, 153464, 38036, 65500, 65525, 65863, 37951, 65175] avg=65536.0000 max=153464 max_rank=1 min=32774 min_rank=0 imbalance_ratio=2.341675 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[62892, 26742, 32772, 115669, 65525, 86348, 67029, 67311] avg=65536.0000 max=115669 max_rank=3 min=26742 min_rank=1 imbalance_ratio=1.764969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[62892, 26742, 32772, 115669, 65525, 86348, 67029, 67311] avg=65536.0000 max=115669 max_rank=3 min=26742 min_rank=1 imbalance_ratio=1.764969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] avg=65536.0000 max=66193 max_rank=6 min=64301 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=87271 max_load_reduction_pct=0.568674 moved_to_replicas=169962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] avg=65536.0000 max=66193 max_rank=6 min=64301 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=87271 max_load_reduction_pct=0.568674 moved_to_replicas=169962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] avg=65536.0000 max=66194 max_rank=5 min=63792 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49475 max_load_reduction_pct=0.427729 moved_to_replicas=138511 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] avg=65536.0000 max=66194 max_rank=5 min=63792 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49475 max_load_reduction_pct=0.427729 moved_to_replicas=138511 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[65540, 66191, 64301, 65500, 65525, 65863, 66193, 65175] avg=65536.0000 max=66193 max_rank=6 min=64301 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=87271 max_load_reduction_pct=0.568674 moved_to_replicas=169962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] avg=65536.0000 max=66194 max_rank=5 min=63792 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49475 max_load_reduction_pct=0.427729 moved_to_replicas=138511 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65340, 65583, 32887, 98306, 32770, 32899, 98178] avg=65536.0000 max=98325 max_rank=0 min=32770 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] avg=65536.0000 max=66194 max_rank=5 min=63792 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49475 max_load_reduction_pct=0.427729 moved_to_replicas=138511 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[62892, 26742, 32772, 115669, 65525, 86348, 67029, 67311] avg=65536.0000 max=115669 max_rank=3 min=26742 min_rank=1 imbalance_ratio=1.764969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[62892, 26742, 32772, 115669, 65525, 86348, 67029, 67311] avg=65536.0000 max=115669 max_rank=3 min=26742 min_rank=1 imbalance_ratio=1.764969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65340, 65583, 32887, 98306, 32770, 32899, 98178] avg=65536.0000 max=98325 max_rank=0 min=32770 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65340, 65583, 32887, 98306, 32770, 32899, 98178] avg=65536.0000 max=98325 max_rank=0 min=32770 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] avg=65536.0000 max=66192 max_rank=0 min=64885 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191940 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65340, 65583, 32887, 98306, 32770, 32899, 98178] avg=65536.0000 max=98325 max_rank=0 min=32770 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[62892, 26742, 32772, 115669, 65525, 86348, 67029, 67311] avg=65536.0000 max=115669 max_rank=3 min=26742 min_rank=1 imbalance_ratio=1.764969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] avg=65536.0000 max=66194 max_rank=5 min=63792 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49475 max_load_reduction_pct=0.427729 moved_to_replicas=138511 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65340, 65583, 32887, 98306, 32770, 32899, 98178] avg=65536.0000 max=98325 max_rank=0 min=32770 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] avg=65536.0000 max=66194 max_rank=5 min=63792 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49475 max_load_reduction_pct=0.427729 moved_to_replicas=138511 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] avg=65536.0000 max=66192 max_rank=0 min=64885 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191940 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] avg=65536.0000 max=66192 max_rank=0 min=64885 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191940 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] avg=65536.0000 max=66192 max_rank=0 min=64885 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191940 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[64011, 63792, 66191, 66191, 65525, 66194, 66192, 66192] avg=65536.0000 max=66194 max_rank=5 min=63792 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=49475 max_load_reduction_pct=0.427729 moved_to_replicas=138511 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[98231, 32818, 66342, 98273, 36169, 65340, 94324, 32791] avg=65536.0000 max=98273 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] avg=65536.0000 max=66192 max_rank=0 min=64885 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191940 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65340, 65583, 32887, 98306, 32770, 32899, 98178] avg=65536.0000 max=98325 max_rank=0 min=32770 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65340, 65583, 32887, 98306, 32770, 32899, 98178] avg=65536.0000 max=98325 max_rank=0 min=32770 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[98231, 32818, 66342, 98273, 36169, 65340, 94324, 32791] avg=65536.0000 max=98273 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[98231, 32818, 66342, 98273, 36169, 65340, 94324, 32791] avg=65536.0000 max=98273 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] avg=65536.0000 max=66192 max_rank=0 min=64451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32081 max_load_reduction_pct=0.326448 moved_to_replicas=181915 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[98231, 32818, 66342, 98273, 36169, 65340, 94324, 32791] avg=65536.0000 max=98273 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[98325, 65340, 65583, 32887, 98306, 32770, 32899, 98178] avg=65536.0000 max=98325 max_rank=0 min=32770 min_rank=5 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] avg=65536.0000 max=66192 max_rank=0 min=64885 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191940 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[98231, 32818, 66342, 98273, 36169, 65340, 94324, 32791] avg=65536.0000 max=98273 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] avg=65536.0000 max=66192 max_rank=0 min=64885 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191940 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] avg=65536.0000 max=66192 max_rank=0 min=64451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32081 max_load_reduction_pct=0.326448 moved_to_replicas=181915 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] avg=65536.0000 max=66192 max_rank=0 min=64451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32081 max_load_reduction_pct=0.326448 moved_to_replicas=181915 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] avg=65536.0000 max=66192 max_rank=0 min=64451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32081 max_load_reduction_pct=0.326448 moved_to_replicas=181915 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[66192, 65340, 65583, 65001, 66192, 64903, 64885, 66192] avg=65536.0000 max=66192 max_rank=0 min=64885 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32133 max_load_reduction_pct=0.326804 moved_to_replicas=191940 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 91655, 72147, 97886, 3, 65632, 65416] avg=65536.0000 max=97886 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] avg=65536.0000 max=66192 max_rank=0 min=64451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32081 max_load_reduction_pct=0.326448 moved_to_replicas=181915 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[98231, 32818, 66342, 98273, 36169, 65340, 94324, 32791] avg=65536.0000 max=98273 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 91655, 72147, 97886, 3, 65632, 65416] avg=65536.0000 max=97886 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[98231, 32818, 66342, 98273, 36169, 65340, 94324, 32791] avg=65536.0000 max=98273 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] avg=65536.0000 max=66194 max_rank=2 min=63113 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31692 max_load_reduction_pct=0.323764 moved_to_replicas=126016 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 91655, 72147, 97886, 3, 65632, 65416] avg=65536.0000 max=97886 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 91655, 72147, 97886, 3, 65632, 65416] avg=65536.0000 max=97886 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[98231, 32818, 66342, 98273, 36169, 65340, 94324, 32791] avg=65536.0000 max=98273 max_rank=3 min=32791 min_rank=7 imbalance_ratio=1.499527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] avg=65536.0000 max=66192 max_rank=0 min=64451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32081 max_load_reduction_pct=0.326448 moved_to_replicas=181915 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] avg=65536.0000 max=66194 max_rank=2 min=63113 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31692 max_load_reduction_pct=0.323764 moved_to_replicas=126016 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 91655, 72147, 97886, 3, 65632, 65416] avg=65536.0000 max=97886 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] avg=65536.0000 max=66192 max_rank=0 min=64451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32081 max_load_reduction_pct=0.326448 moved_to_replicas=181915 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] avg=65536.0000 max=66194 max_rank=2 min=63113 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31692 max_load_reduction_pct=0.323764 moved_to_replicas=126016 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] avg=65536.0000 max=66194 max_rank=2 min=63113 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31692 max_load_reduction_pct=0.323764 moved_to_replicas=126016 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[66192, 64857, 66192, 66192, 64451, 65340, 66192, 64872] avg=65536.0000 max=66192 max_rank=0 min=64451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32081 max_load_reduction_pct=0.326448 moved_to_replicas=181915 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65273, 98549, 168371, 10, 65530, 32825, 28662, 65068] avg=65536.0000 max=168371 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.569138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] avg=65536.0000 max=66194 max_rank=2 min=63113 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31692 max_load_reduction_pct=0.323764 moved_to_replicas=126016 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 91655, 72147, 97886, 3, 65632, 65416] avg=65536.0000 max=97886 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65273, 98549, 168371, 10, 65530, 32825, 28662, 65068] avg=65536.0000 max=168371 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.569138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65273, 98549, 168371, 10, 65530, 32825, 28662, 65068] avg=65536.0000 max=168371 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.569138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65273, 98549, 168371, 10, 65530, 32825, 28662, 65068] avg=65536.0000 max=168371 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.569138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] avg=65536.0000 max=66192 max_rank=1 min=64300 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102179 max_load_reduction_pct=0.606868 moved_to_replicas=275378 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 91655, 72147, 97886, 3, 65632, 65416] avg=65536.0000 max=97886 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 91655, 72147, 97886, 3, 65632, 65416] avg=65536.0000 max=97886 max_rank=4 min=3 min_rank=5 imbalance_ratio=1.493622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] avg=65536.0000 max=66194 max_rank=2 min=63113 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31692 max_load_reduction_pct=0.323764 moved_to_replicas=126016 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] avg=65536.0000 max=66192 max_rank=1 min=64300 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102179 max_load_reduction_pct=0.606868 moved_to_replicas=275378 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65273, 98549, 168371, 10, 65530, 32825, 28662, 65068] avg=65536.0000 max=168371 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.569138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] avg=65536.0000 max=66192 max_rank=1 min=64300 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102179 max_load_reduction_pct=0.606868 moved_to_replicas=275378 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] avg=65536.0000 max=66192 max_rank=1 min=64300 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102179 max_load_reduction_pct=0.606868 moved_to_replicas=275378 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] avg=65536.0000 max=66194 max_rank=2 min=63113 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31692 max_load_reduction_pct=0.323764 moved_to_replicas=126016 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65872, 65677, 66194, 66192, 66192, 63113, 65632, 65416] avg=65536.0000 max=66194 max_rank=2 min=63113 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31692 max_load_reduction_pct=0.323764 moved_to_replicas=126016 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[23619, 39757, 98283, 96717, 98325, 65534, 3900, 98153] avg=65536.0000 max=98325 max_rank=4 min=3900 min_rank=6 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65273, 98549, 168371, 10, 65530, 32825, 28662, 65068] avg=65536.0000 max=168371 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.569138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] avg=65536.0000 max=66192 max_rank=1 min=64300 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102179 max_load_reduction_pct=0.606868 moved_to_replicas=275378 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[23619, 39757, 98283, 96717, 98325, 65534, 3900, 98153] avg=65536.0000 max=98325 max_rank=4 min=3900 min_rank=6 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[23619, 39757, 98283, 96717, 98325, 65534, 3900, 98153] avg=65536.0000 max=98325 max_rank=4 min=3900 min_rank=6 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[23619, 39757, 98283, 96717, 98325, 65534, 3900, 98153] avg=65536.0000 max=98325 max_rank=4 min=3900 min_rank=6 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=61600 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.326784 moved_to_replicas=251098 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] avg=65536.0000 max=66192 max_rank=1 min=64300 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102179 max_load_reduction_pct=0.606868 moved_to_replicas=275378 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65273, 98549, 168371, 10, 65530, 32825, 28662, 65068] avg=65536.0000 max=168371 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.569138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65273, 98549, 168371, 10, 65530, 32825, 28662, 65068] avg=65536.0000 max=168371 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.569138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=61600 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.326784 moved_to_replicas=251098 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=61600 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.326784 moved_to_replicas=251098 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[23619, 39757, 98283, 96717, 98325, 65534, 3900, 98153] avg=65536.0000 max=98325 max_rank=4 min=3900 min_rank=6 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=61600 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.326784 moved_to_replicas=251098 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] avg=65536.0000 max=66192 max_rank=1 min=64300 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102179 max_load_reduction_pct=0.606868 moved_to_replicas=275378 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[56797, 108180, 29, 32767, 65576, 62263, 100463, 98213] avg=65536.0000 max=108180 max_rank=1 min=29 min_rank=2 imbalance_ratio=1.650696 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65273, 66192, 66192, 65541, 65530, 66192, 64300, 65068] avg=65536.0000 max=66192 max_rank=1 min=64300 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=102179 max_load_reduction_pct=0.606868 moved_to_replicas=275378 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[23619, 39757, 98283, 96717, 98325, 65534, 3900, 98153] avg=65536.0000 max=98325 max_rank=4 min=3900 min_rank=6 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=61600 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.326784 moved_to_replicas=251098 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[56797, 108180, 29, 32767, 65576, 62263, 100463, 98213] avg=65536.0000 max=108180 max_rank=1 min=29 min_rank=2 imbalance_ratio=1.650696 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[56797, 108180, 29, 32767, 65576, 62263, 100463, 98213] avg=65536.0000 max=108180 max_rank=1 min=29 min_rank=2 imbalance_ratio=1.650696 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=62263 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41986 max_load_reduction_pct=0.388112 moved_to_replicas=217585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[56797, 108180, 29, 32767, 65576, 62263, 100463, 98213] avg=65536.0000 max=108180 max_rank=1 min=29 min_rank=2 imbalance_ratio=1.650696 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=61600 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.326784 moved_to_replicas=251098 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[23619, 39757, 98283, 96717, 98325, 65534, 3900, 98153] avg=65536.0000 max=98325 max_rank=4 min=3900 min_rank=6 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=62263 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41986 max_load_reduction_pct=0.388112 moved_to_replicas=217585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[23619, 39757, 98283, 96717, 98325, 65534, 3900, 98153] avg=65536.0000 max=98325 max_rank=4 min=3900 min_rank=6 imbalance_ratio=1.500320 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=62263 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41986 max_load_reduction_pct=0.388112 moved_to_replicas=217585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=62263 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41986 max_load_reduction_pct=0.388112 moved_to_replicas=217585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[56797, 108180, 29, 32767, 65576, 62263, 100463, 98213] avg=65536.0000 max=108180 max_rank=1 min=29 min_rank=2 imbalance_ratio=1.650696 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[25902, 98292, 73067, 64927, 32770, 32733, 98278, 98319] avg=65536.0000 max=98319 max_rank=7 min=25902 min_rank=0 imbalance_ratio=1.500229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=61600 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.326784 moved_to_replicas=251098 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[61600, 66194, 66192, 66192, 66192, 65534, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=61600 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.326784 moved_to_replicas=251098 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[56797, 108180, 29, 32767, 65576, 62263, 100463, 98213] avg=65536.0000 max=108180 max_rank=1 min=29 min_rank=2 imbalance_ratio=1.650696 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=62263 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41986 max_load_reduction_pct=0.388112 moved_to_replicas=217585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[25902, 98292, 73067, 64927, 32770, 32733, 98278, 98319] avg=65536.0000 max=98319 max_rank=7 min=25902 min_rank=0 imbalance_ratio=1.500229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[25902, 98292, 73067, 64927, 32770, 32733, 98278, 98319] avg=65536.0000 max=98319 max_rank=7 min=25902 min_rank=0 imbalance_ratio=1.500229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=64833 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=32126 max_load_reduction_pct=0.326753 moved_to_replicas=210375 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[25902, 98292, 73067, 64927, 32770, 32733, 98278, 98319] avg=65536.0000 max=98319 max_rank=7 min=25902 min_rank=0 imbalance_ratio=1.500229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=62263 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41986 max_load_reduction_pct=0.388112 moved_to_replicas=217585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[56797, 108180, 29, 32767, 65576, 62263, 100463, 98213] avg=65536.0000 max=108180 max_rank=1 min=29 min_rank=2 imbalance_ratio=1.650696 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=64833 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=32126 max_load_reduction_pct=0.326753 moved_to_replicas=210375 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[56797, 108180, 29, 32767, 65576, 62263, 100463, 98213] avg=65536.0000 max=108180 max_rank=1 min=29 min_rank=2 imbalance_ratio=1.650696 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=64833 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=32126 max_load_reduction_pct=0.326753 moved_to_replicas=210375 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=64833 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=32126 max_load_reduction_pct=0.326753 moved_to_replicas=210375 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[25902, 98292, 73067, 64927, 32770, 32733, 98278, 98319] avg=65536.0000 max=98319 max_rank=7 min=25902 min_rank=0 imbalance_ratio=1.500229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[63481, 97781, 70972, 94928, 65559, 65581, 32771, 33215] avg=65536.0000 max=97781 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=62263 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41986 max_load_reduction_pct=0.388112 moved_to_replicas=217585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[25902, 98292, 73067, 64927, 32770, 32733, 98278, 98319] avg=65536.0000 max=98319 max_rank=7 min=25902 min_rank=0 imbalance_ratio=1.500229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[66118, 66194, 65561, 66192, 65576, 62263, 66192, 66192] avg=65536.0000 max=66194 max_rank=1 min=62263 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=41986 max_load_reduction_pct=0.388112 moved_to_replicas=217585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=64833 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=32126 max_load_reduction_pct=0.326753 moved_to_replicas=210375 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[63481, 97781, 70972, 94928, 65559, 65581, 32771, 33215] avg=65536.0000 max=97781 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] avg=65536.0000 max=66193 max_rank=7 min=64020 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31588 max_load_reduction_pct=0.323048 moved_to_replicas=130212 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[63481, 97781, 70972, 94928, 65559, 65581, 32771, 33215] avg=65536.0000 max=97781 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[63481, 97781, 70972, 94928, 65559, 65581, 32771, 33215] avg=65536.0000 max=97781 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=64833 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=32126 max_load_reduction_pct=0.326753 moved_to_replicas=210375 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] avg=65536.0000 max=66193 max_rank=7 min=64020 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31588 max_load_reduction_pct=0.323048 moved_to_replicas=130212 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] avg=65536.0000 max=66193 max_rank=7 min=64020 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31588 max_load_reduction_pct=0.323048 moved_to_replicas=130212 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[25902, 98292, 73067, 64927, 32770, 32733, 98278, 98319] avg=65536.0000 max=98319 max_rank=7 min=25902 min_rank=0 imbalance_ratio=1.500229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] avg=65536.0000 max=66193 max_rank=7 min=64020 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31588 max_load_reduction_pct=0.323048 moved_to_replicas=130212 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[25902, 98292, 73067, 64927, 32770, 32733, 98278, 98319] avg=65536.0000 max=98319 max_rank=7 min=25902 min_rank=0 imbalance_ratio=1.500229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[63481, 97781, 70972, 94928, 65559, 65581, 32771, 33215] avg=65536.0000 max=97781 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96968, 130333, 65576, 65538, 92098, 40995] avg=65536.0000 max=130333 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.988724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=64833 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=32126 max_load_reduction_pct=0.326753 moved_to_replicas=210375 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[63481, 97781, 70972, 94928, 65559, 65581, 32771, 33215] avg=65536.0000 max=97781 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[64903, 66192, 66193, 64927, 64856, 64833, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=64833 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=32126 max_load_reduction_pct=0.326753 moved_to_replicas=210375 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] avg=65536.0000 max=66193 max_rank=7 min=64020 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31588 max_load_reduction_pct=0.323048 moved_to_replicas=130212 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96968, 130333, 65576, 65538, 92098, 40995] avg=65536.0000 max=130333 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.988724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96968, 130333, 65576, 65538, 92098, 40995] avg=65536.0000 max=130333 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.988724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=64143 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64139 max_load_reduction_pct=0.492116 moved_to_replicas=242275 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96968, 130333, 65576, 65538, 92098, 40995] avg=65536.0000 max=130333 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.988724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] avg=65536.0000 max=66193 max_rank=7 min=64020 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31588 max_load_reduction_pct=0.323048 moved_to_replicas=130212 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=64143 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64139 max_load_reduction_pct=0.492116 moved_to_replicas=242275 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=64143 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64139 max_load_reduction_pct=0.492116 moved_to_replicas=242275 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[63481, 97781, 70972, 94928, 65559, 65581, 32771, 33215] avg=65536.0000 max=97781 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=64143 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64139 max_load_reduction_pct=0.492116 moved_to_replicas=242275 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96968, 130333, 65576, 65538, 92098, 40995] avg=65536.0000 max=130333 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.988724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[63481, 97781, 70972, 94928, 65559, 65581, 32771, 33215] avg=65536.0000 max=97781 max_rank=1 min=32771 min_rank=6 imbalance_ratio=1.492020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] avg=65536.0000 max=66193 max_rank=7 min=64020 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31588 max_load_reduction_pct=0.323048 moved_to_replicas=130212 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96968, 130333, 65576, 65538, 92098, 40995] avg=65536.0000 max=130333 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.988724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=64143 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64139 max_load_reduction_pct=0.492116 moved_to_replicas=242275 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[64020, 66192, 66191, 66192, 65559, 65581, 64360, 66193] avg=65536.0000 max=66193 max_rank=7 min=64020 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=31588 max_load_reduction_pct=0.323048 moved_to_replicas=130212 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=64143 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64139 max_load_reduction_pct=0.492116 moved_to_replicas=242275 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96968, 130333, 65576, 65538, 92098, 40995] avg=65536.0000 max=130333 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.988724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[13, 32767, 96968, 130333, 65576, 65538, 92098, 40995] avg=65536.0000 max=130333 max_rank=3 min=13 min_rank=0 imbalance_ratio=1.988724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=64143 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64139 max_load_reduction_pct=0.492116 moved_to_replicas=242275 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[64263, 64143, 66192, 66192, 65576, 65538, 66194, 66190] avg=65536.0000 max=66194 max_rank=6 min=64143 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64139 max_load_reduction_pct=0.492116 moved_to_replicas=242275 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=2084514 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=518016 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=2100843 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=500992 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=2101222 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=501376 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=2098309 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=504064 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=2104439 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=496768 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=2102169 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=502656 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=2103567 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=498304 peak_demand=66195 required_capacity=1.010056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=2082153 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=519936 peak_demand=66194 required_capacity=1.010040 + [2026-08-30 05:14:10] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 523.9 | throughput per GPU (TFLOP/s/GPU): 636.3 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.038128E+01 | loss scale: 1.0 | grad norm: 16.223 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=81920 expected_recv=129327 expected_overflow=0 exact_recv_per_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131187 expected_overflow=0 exact_recv_per_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=81920 expected_recv=126143 expected_overflow=0 exact_recv_per_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=81920 expected_recv=129327 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131187 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=81920 expected_recv=126143 actual_valid_recv=127616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=127616 slack_slots=36224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131077 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=81920 expected_recv=128252 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=81920 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131077 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=81920 expected_recv=130044 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131012 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=81920 expected_recv=128252 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=81920 expected_recv=132387 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=81920 expected_recv=130044 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131035 expected_overflow=0 exact_recv_per_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131012 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131035 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=81920 expected_recv=129093 expected_overflow=0 exact_recv_per_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=81920 expected_recv=129093 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130124 expected_overflow=0 exact_recv_per_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131054 expected_overflow=0 exact_recv_per_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130124 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131044 expected_overflow=0 exact_recv_per_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130110 expected_overflow=0 exact_recv_per_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131054 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131044 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130110 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130570 expected_overflow=0 exact_recv_per_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130704 expected_overflow=0 exact_recv_per_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=134144 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130570 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130704 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=81920 expected_recv=129526 expected_overflow=0 exact_recv_per_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=81920 expected_recv=129620 expected_overflow=0 exact_recv_per_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=81920 expected_recv=129526 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=81920 expected_recv=131002 expected_overflow=0 exact_recv_per_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=81920 expected_recv=129620 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=81920 expected_recv=131002 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=81920 expected_recv=128535 expected_overflow=0 exact_recv_per_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131033 expected_overflow=0 exact_recv_per_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=81920 expected_recv=128535 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131033 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130564 expected_overflow=0 exact_recv_per_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130273 expected_overflow=0 exact_recv_per_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130564 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131334 expected_overflow=0 exact_recv_per_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130273 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131334 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=81920 expected_recv=132069 expected_overflow=0 exact_recv_per_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=81920 expected_recv=129780 expected_overflow=0 exact_recv_per_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=81920 expected_recv=132069 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=81920 expected_recv=129780 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=81920 expected_recv=131055 expected_overflow=0 exact_recv_per_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=81920 expected_recv=131055 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=81920 expected_recv=129922 expected_overflow=0 exact_recv_per_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=81920 expected_recv=128285 expected_overflow=0 exact_recv_per_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=81920 expected_recv=129922 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=81920 expected_recv=128285 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=81920 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=81920 expected_recv=125874 expected_overflow=0 exact_recv_per_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=81920 expected_recv=132387 actual_valid_recv=134272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=81920 expected_recv=125874 actual_valid_recv=127360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=134272 slack_slots=29568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131016 expected_overflow=0 exact_recv_per_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=127360 slack_slots=36480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131016 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131059 expected_overflow=0 exact_recv_per_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131091 expected_overflow=0 exact_recv_per_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131059 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=81920 expected_recv=129616 expected_overflow=0 exact_recv_per_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131492 expected_overflow=0 exact_recv_per_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131091 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=81920 expected_recv=129616 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131492 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=134144 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=81920 expected_recv=129587 expected_overflow=0 exact_recv_per_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131008 expected_overflow=0 exact_recv_per_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=81920 expected_recv=129587 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=81920 expected_recv=129563 expected_overflow=0 exact_recv_per_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131008 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=81920 expected_recv=129721 expected_overflow=0 exact_recv_per_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=81920 expected_recv=129563 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=81920 expected_recv=130426 expected_overflow=0 exact_recv_per_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=81920 expected_recv=129721 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=81920 expected_recv=130426 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=81920 expected_recv=130416 expected_overflow=0 exact_recv_per_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=81920 expected_recv=130416 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=81920 expected_recv=129810 expected_overflow=0 exact_recv_per_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131156 expected_overflow=0 exact_recv_per_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131209 expected_overflow=0 exact_recv_per_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=81920 expected_recv=129810 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131156 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131137 expected_overflow=0 exact_recv_per_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131209 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131137 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=81920 expected_recv=128060 expected_overflow=0 exact_recv_per_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=81920 expected_recv=128060 actual_valid_recv=128896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=81920 expected_recv=128687 expected_overflow=0 exact_recv_per_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=128896 slack_slots=34944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=81920 expected_recv=128687 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=81920 expected_recv=129668 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=81920 expected_recv=129668 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=81920 expected_recv=129647 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131117 expected_overflow=0 exact_recv_per_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=81920 expected_recv=129988 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=81920 expected_recv=129737 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=81920 expected_recv=129647 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=81920 expected_recv=129689 expected_overflow=0 exact_recv_per_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131117 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=81920 expected_recv=129988 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=81920 expected_recv=129737 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=81920 expected_recv=129689 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=81920 expected_recv=127141 expected_overflow=0 exact_recv_per_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=81920 expected_recv=127141 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=128768 slack_slots=35072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131096 expected_overflow=0 exact_recv_per_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131106 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=81920 expected_recv=129815 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131096 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131106 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=81920 expected_recv=129815 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131637 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131637 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=81920 expected_recv=129700 expected_overflow=0 exact_recv_per_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=81920 expected_recv=129700 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=81920 expected_recv=129349 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=81920 expected_recv=129517 expected_overflow=0 exact_recv_per_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=81920 expected_recv=129349 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=81920 expected_recv=129517 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131547 expected_overflow=0 exact_recv_per_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131547 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130582 expected_overflow=0 exact_recv_per_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=81920 expected_recv=129496 expected_overflow=0 exact_recv_per_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=81920 expected_recv=131067 expected_overflow=0 exact_recv_per_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130582 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131054 expected_overflow=0 exact_recv_per_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=81920 expected_recv=129496 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=81920 expected_recv=129627 expected_overflow=0 exact_recv_per_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=81920 expected_recv=131067 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131054 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=81920 expected_recv=129627 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=81920 expected_recv=128885 expected_overflow=0 exact_recv_per_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=81920 expected_recv=128885 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=81920 expected_recv=129791 expected_overflow=0 exact_recv_per_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130630 expected_overflow=0 exact_recv_per_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=81920 expected_recv=129791 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=81920 expected_recv=131100 expected_overflow=0 exact_recv_per_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130630 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=81920 expected_recv=131100 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131095 expected_overflow=0 exact_recv_per_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131095 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=81920 expected_recv=127719 expected_overflow=0 exact_recv_per_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=81920 expected_recv=127719 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[24404, 80468, 88028, 45036, 90526, 68658, 50602, 76566] avg=65536.0000 max=90526 max_rank=4 min=24404 min_rank=0 imbalance_ratio=1.381317 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=128768 slack_slots=35072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130861 expected_overflow=0 exact_recv_per_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130861 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131118 expected_overflow=0 exact_recv_per_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[24404, 80468, 88028, 45036, 90526, 68658, 50602, 76566] avg=65536.0000 max=90526 max_rank=4 min=24404 min_rank=0 imbalance_ratio=1.381317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[24404, 80468, 88028, 45036, 90526, 68658, 50602, 76566] avg=65536.0000 max=90526 max_rank=4 min=24404 min_rank=0 imbalance_ratio=1.381317 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] avg=65536.0000 max=66193 max_rank=7 min=63442 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24333 max_load_reduction_pct=0.268796 moved_to_replicas=146111 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131118 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[24404, 80468, 88028, 45036, 90526, 68658, 50602, 76566] avg=65536.0000 max=90526 max_rank=4 min=24404 min_rank=0 imbalance_ratio=1.381317 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[24404, 80468, 88028, 45036, 90526, 68658, 50602, 76566] avg=65536.0000 max=90526 max_rank=4 min=24404 min_rank=0 imbalance_ratio=1.381317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] avg=65536.0000 max=66193 max_rank=7 min=63442 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24333 max_load_reduction_pct=0.268796 moved_to_replicas=146111 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] avg=65536.0000 max=66193 max_rank=7 min=63442 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24333 max_load_reduction_pct=0.268796 moved_to_replicas=146111 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] avg=65536.0000 max=66193 max_rank=7 min=63442 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24333 max_load_reduction_pct=0.268796 moved_to_replicas=146111 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98463, 32831, 83162, 113148, 65549, 65511, 32839] avg=65536.0000 max=113148 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[24404, 80468, 88028, 45036, 90526, 68658, 50602, 76566] avg=65536.0000 max=90526 max_rank=4 min=24404 min_rank=0 imbalance_ratio=1.381317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] avg=65536.0000 max=66193 max_rank=7 min=63442 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24333 max_load_reduction_pct=0.268796 moved_to_replicas=146111 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[24404, 80468, 88028, 45036, 90526, 68658, 50602, 76566] avg=65536.0000 max=90526 max_rank=4 min=24404 min_rank=0 imbalance_ratio=1.381317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98463, 32831, 83162, 113148, 65549, 65511, 32839] avg=65536.0000 max=113148 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] avg=65536.0000 max=66194 max_rank=3 min=64026 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46954 max_load_reduction_pct=0.414979 moved_to_replicas=192440 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98463, 32831, 83162, 113148, 65549, 65511, 32839] avg=65536.0000 max=113148 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98463, 32831, 83162, 113148, 65549, 65511, 32839] avg=65536.0000 max=113148 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] avg=65536.0000 max=66193 max_rank=7 min=63442 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24333 max_load_reduction_pct=0.268796 moved_to_replicas=146111 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] avg=65536.0000 max=66193 max_rank=7 min=63442 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24333 max_load_reduction_pct=0.268796 moved_to_replicas=146111 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98463, 32831, 83162, 113148, 65549, 65511, 32839] avg=65536.0000 max=113148 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] avg=65536.0000 max=66194 max_rank=3 min=64026 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46954 max_load_reduction_pct=0.414979 moved_to_replicas=192440 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] avg=65536.0000 max=66194 max_rank=3 min=64026 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46954 max_load_reduction_pct=0.414979 moved_to_replicas=192440 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] avg=65536.0000 max=66194 max_rank=3 min=64026 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46954 max_load_reduction_pct=0.414979 moved_to_replicas=192440 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[97264, 33154, 32783, 32774, 65190, 116849, 65531, 80743] avg=65536.0000 max=116849 max_rank=5 min=32774 min_rank=3 imbalance_ratio=1.782974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] avg=65536.0000 max=66194 max_rank=3 min=64026 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46954 max_load_reduction_pct=0.414979 moved_to_replicas=192440 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98463, 32831, 83162, 113148, 65549, 65511, 32839] avg=65536.0000 max=113148 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98463, 32831, 83162, 113148, 65549, 65511, 32839] avg=65536.0000 max=113148 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[24404, 80468, 88028, 45036, 90526, 68658, 50602, 76566] avg=65536.0000 max=90526 max_rank=4 min=24404 min_rank=0 imbalance_ratio=1.381317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[97264, 33154, 32783, 32774, 65190, 116849, 65531, 80743] avg=65536.0000 max=116849 max_rank=5 min=32774 min_rank=3 imbalance_ratio=1.782974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] avg=65536.0000 max=66192 max_rank=0 min=64226 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50657 max_load_reduction_pct=0.433525 moved_to_replicas=192466 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[97264, 33154, 32783, 32774, 65190, 116849, 65531, 80743] avg=65536.0000 max=116849 max_rank=5 min=32774 min_rank=3 imbalance_ratio=1.782974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[97264, 33154, 32783, 32774, 65190, 116849, 65531, 80743] avg=65536.0000 max=116849 max_rank=5 min=32774 min_rank=3 imbalance_ratio=1.782974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] avg=65536.0000 max=66194 max_rank=3 min=64026 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46954 max_load_reduction_pct=0.414979 moved_to_replicas=192440 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] avg=65536.0000 max=66194 max_rank=3 min=64026 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46954 max_load_reduction_pct=0.414979 moved_to_replicas=192440 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[97264, 33154, 32783, 32774, 65190, 116849, 65531, 80743] avg=65536.0000 max=116849 max_rank=5 min=32774 min_rank=3 imbalance_ratio=1.782974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] avg=65536.0000 max=66192 max_rank=0 min=64226 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50657 max_load_reduction_pct=0.433525 moved_to_replicas=192466 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] avg=65536.0000 max=66192 max_rank=0 min=64226 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50657 max_load_reduction_pct=0.433525 moved_to_replicas=192466 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] avg=65536.0000 max=66192 max_rank=0 min=64226 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50657 max_load_reduction_pct=0.433525 moved_to_replicas=192466 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32529, 65400, 61358, 48916, 65495, 94603, 110990, 44997] avg=65536.0000 max=110990 max_rank=6 min=32529 min_rank=0 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[63695, 66191, 66192, 66192, 66192, 66191, 63442, 66193] avg=65536.0000 max=66193 max_rank=7 min=63442 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=24333 max_load_reduction_pct=0.268796 moved_to_replicas=146111 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] avg=65536.0000 max=66192 max_rank=0 min=64226 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50657 max_load_reduction_pct=0.433525 moved_to_replicas=192466 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[97264, 33154, 32783, 32774, 65190, 116849, 65531, 80743] avg=65536.0000 max=116849 max_rank=5 min=32774 min_rank=3 imbalance_ratio=1.782974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[97264, 33154, 32783, 32774, 65190, 116849, 65531, 80743] avg=65536.0000 max=116849 max_rank=5 min=32774 min_rank=3 imbalance_ratio=1.782974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] avg=65536.0000 max=66194 max_rank=3 min=63335 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=44796 max_load_reduction_pct=0.403604 moved_to_replicas=148923 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32529, 65400, 61358, 48916, 65495, 94603, 110990, 44997] avg=65536.0000 max=110990 max_rank=6 min=32529 min_rank=0 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32529, 65400, 61358, 48916, 65495, 94603, 110990, 44997] avg=65536.0000 max=110990 max_rank=6 min=32529 min_rank=0 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32529, 65400, 61358, 48916, 65495, 94603, 110990, 44997] avg=65536.0000 max=110990 max_rank=6 min=32529 min_rank=0 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] avg=65536.0000 max=66192 max_rank=0 min=64226 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50657 max_load_reduction_pct=0.433525 moved_to_replicas=192466 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] avg=65536.0000 max=66192 max_rank=0 min=64226 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50657 max_load_reduction_pct=0.433525 moved_to_replicas=192466 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32529, 65400, 61358, 48916, 65495, 94603, 110990, 44997] avg=65536.0000 max=110990 max_rank=6 min=32529 min_rank=0 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] avg=65536.0000 max=66194 max_rank=3 min=63335 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=44796 max_load_reduction_pct=0.403604 moved_to_replicas=148923 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] avg=65536.0000 max=66194 max_rank=3 min=63335 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=44796 max_load_reduction_pct=0.403604 moved_to_replicas=148923 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] avg=65536.0000 max=66194 max_rank=3 min=63335 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=44796 max_load_reduction_pct=0.403604 moved_to_replicas=148923 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 98303, 131047, 33766, 32831, 33137] avg=65536.0000 max=131047 max_rank=4 min=32831 min_rank=6 imbalance_ratio=1.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32785, 98463, 32831, 83162, 113148, 65549, 65511, 32839] avg=65536.0000 max=113148 max_rank=4 min=32785 min_rank=0 imbalance_ratio=1.726501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] avg=65536.0000 max=66194 max_rank=3 min=63335 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=44796 max_load_reduction_pct=0.403604 moved_to_replicas=148923 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32529, 65400, 61358, 48916, 65495, 94603, 110990, 44997] avg=65536.0000 max=110990 max_rank=6 min=32529 min_rank=0 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] avg=65536.0000 max=66192 max_rank=3 min=64390 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64855 max_load_reduction_pct=0.494899 moved_to_replicas=193899 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32529, 65400, 61358, 48916, 65495, 94603, 110990, 44997] avg=65536.0000 max=110990 max_rank=6 min=32529 min_rank=0 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 98303, 131047, 33766, 32831, 33137] avg=65536.0000 max=131047 max_rank=4 min=32831 min_rank=6 imbalance_ratio=1.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 98303, 131047, 33766, 32831, 33137] avg=65536.0000 max=131047 max_rank=4 min=32831 min_rank=6 imbalance_ratio=1.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[65513, 66192, 64026, 66194, 66193, 65549, 65511, 65110] avg=65536.0000 max=66194 max_rank=3 min=64026 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=46954 max_load_reduction_pct=0.414979 moved_to_replicas=192440 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 98303, 131047, 33766, 32831, 33137] avg=65536.0000 max=131047 max_rank=4 min=32831 min_rank=6 imbalance_ratio=1.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] avg=65536.0000 max=66194 max_rank=3 min=63335 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=44796 max_load_reduction_pct=0.403604 moved_to_replicas=148923 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 98303, 131047, 33766, 32831, 33137] avg=65536.0000 max=131047 max_rank=4 min=32831 min_rank=6 imbalance_ratio=1.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] avg=65536.0000 max=66194 max_rank=3 min=63335 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=44796 max_load_reduction_pct=0.403604 moved_to_replicas=148923 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] avg=65536.0000 max=66192 max_rank=3 min=64390 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64855 max_load_reduction_pct=0.494899 moved_to_replicas=193899 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] avg=65536.0000 max=66192 max_rank=3 min=64390 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64855 max_load_reduction_pct=0.494899 moved_to_replicas=193899 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[10354, 71152, 65526, 98302, 86662, 126685, 32750, 32857] avg=65536.0000 max=126685 max_rank=5 min=10354 min_rank=0 imbalance_ratio=1.933060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] avg=65536.0000 max=66192 max_rank=3 min=64390 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64855 max_load_reduction_pct=0.494899 moved_to_replicas=193899 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] avg=65536.0000 max=66192 max_rank=3 min=64390 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64855 max_load_reduction_pct=0.494899 moved_to_replicas=193899 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 98303, 131047, 33766, 32831, 33137] avg=65536.0000 max=131047 max_rank=4 min=32831 min_rank=6 imbalance_ratio=1.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[97264, 33154, 32783, 32774, 65190, 116849, 65531, 80743] avg=65536.0000 max=116849 max_rank=5 min=32774 min_rank=3 imbalance_ratio=1.782974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] avg=65536.0000 max=66193 max_rank=5 min=63587 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60492 max_load_reduction_pct=0.477499 moved_to_replicas=230000 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 98303, 131047, 33766, 32831, 33137] avg=65536.0000 max=131047 max_rank=4 min=32831 min_rank=6 imbalance_ratio=1.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[10354, 71152, 65526, 98302, 86662, 126685, 32750, 32857] avg=65536.0000 max=126685 max_rank=5 min=10354 min_rank=0 imbalance_ratio=1.933060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[10354, 71152, 65526, 98302, 86662, 126685, 32750, 32857] avg=65536.0000 max=126685 max_rank=5 min=10354 min_rank=0 imbalance_ratio=1.933060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[10354, 71152, 65526, 98302, 86662, 126685, 32750, 32857] avg=65536.0000 max=126685 max_rank=5 min=10354 min_rank=0 imbalance_ratio=1.933060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] avg=65536.0000 max=66192 max_rank=3 min=64390 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64855 max_load_reduction_pct=0.494899 moved_to_replicas=193899 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[66192, 64226, 65233, 65533, 65190, 66191, 65531, 66192] avg=65536.0000 max=66192 max_rank=0 min=64226 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=50657 max_load_reduction_pct=0.433525 moved_to_replicas=192466 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[10354, 71152, 65526, 98302, 86662, 126685, 32750, 32857] avg=65536.0000 max=126685 max_rank=5 min=10354 min_rank=0 imbalance_ratio=1.933060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] avg=65536.0000 max=66192 max_rank=3 min=64390 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64855 max_load_reduction_pct=0.494899 moved_to_replicas=193899 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] avg=65536.0000 max=66193 max_rank=5 min=63587 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60492 max_load_reduction_pct=0.477499 moved_to_replicas=230000 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] avg=65536.0000 max=66193 max_rank=5 min=63587 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60492 max_load_reduction_pct=0.477499 moved_to_replicas=230000 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32787, 27170, 65521, 98288, 32785, 55380, 65528, 146829] avg=65536.0000 max=146829 max_rank=7 min=27170 min_rank=1 imbalance_ratio=2.240433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] avg=65536.0000 max=66193 max_rank=5 min=63587 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60492 max_load_reduction_pct=0.477499 moved_to_replicas=230000 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] avg=65536.0000 max=66193 max_rank=5 min=63587 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60492 max_load_reduction_pct=0.477499 moved_to_replicas=230000 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[10354, 71152, 65526, 98302, 86662, 126685, 32750, 32857] avg=65536.0000 max=126685 max_rank=5 min=10354 min_rank=0 imbalance_ratio=1.933060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32529, 65400, 61358, 48916, 65495, 94603, 110990, 44997] avg=65536.0000 max=110990 max_rank=6 min=32529 min_rank=0 imbalance_ratio=1.693573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] avg=65536.0000 max=66195 max_rank=0 min=62930 min_rank=1 imbalance_ratio=1.010056 max_load_reduction=80634 max_load_reduction_pct=0.549169 moved_to_replicas=223966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[10354, 71152, 65526, 98302, 86662, 126685, 32750, 32857] avg=65536.0000 max=126685 max_rank=5 min=10354 min_rank=0 imbalance_ratio=1.933060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32787, 27170, 65521, 98288, 32785, 55380, 65528, 146829] avg=65536.0000 max=146829 max_rank=7 min=27170 min_rank=1 imbalance_ratio=2.240433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] avg=65536.0000 max=66193 max_rank=5 min=63587 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60492 max_load_reduction_pct=0.477499 moved_to_replicas=230000 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32787, 27170, 65521, 98288, 32785, 55380, 65528, 146829] avg=65536.0000 max=146829 max_rank=7 min=27170 min_rank=1 imbalance_ratio=2.240433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32787, 27170, 65521, 98288, 32785, 55380, 65528, 146829] avg=65536.0000 max=146829 max_rank=7 min=27170 min_rank=1 imbalance_ratio=2.240433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32787, 27170, 65521, 98288, 32785, 55380, 65528, 146829] avg=65536.0000 max=146829 max_rank=7 min=27170 min_rank=1 imbalance_ratio=2.240433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[65290, 65400, 63335, 66194, 65495, 66190, 66194, 66190] avg=65536.0000 max=66194 max_rank=3 min=63335 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=44796 max_load_reduction_pct=0.403604 moved_to_replicas=148923 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] avg=65536.0000 max=66195 max_rank=0 min=62930 min_rank=1 imbalance_ratio=1.010056 max_load_reduction=80634 max_load_reduction_pct=0.549169 moved_to_replicas=223966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] avg=65536.0000 max=66193 max_rank=5 min=63587 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60492 max_load_reduction_pct=0.477499 moved_to_replicas=230000 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[20095, 32967, 131017, 65523, 131042, 27, 13142, 130475] avg=65536.0000 max=131042 max_rank=4 min=27 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] avg=65536.0000 max=66195 max_rank=0 min=62930 min_rank=1 imbalance_ratio=1.010056 max_load_reduction=80634 max_load_reduction_pct=0.549169 moved_to_replicas=223966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] avg=65536.0000 max=66195 max_rank=0 min=62930 min_rank=1 imbalance_ratio=1.010056 max_load_reduction=80634 max_load_reduction_pct=0.549169 moved_to_replicas=223966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] avg=65536.0000 max=66195 max_rank=0 min=62930 min_rank=1 imbalance_ratio=1.010056 max_load_reduction=80634 max_load_reduction_pct=0.549169 moved_to_replicas=223966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32787, 27170, 65521, 98288, 32785, 55380, 65528, 146829] avg=65536.0000 max=146829 max_rank=7 min=27170 min_rank=1 imbalance_ratio=2.240433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63424 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=387400 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 98303, 131047, 33766, 32831, 33137] avg=65536.0000 max=131047 max_rank=4 min=32831 min_rank=6 imbalance_ratio=1.999619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[20095, 32967, 131017, 65523, 131042, 27, 13142, 130475] avg=65536.0000 max=131042 max_rank=4 min=27 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32787, 27170, 65521, 98288, 32785, 55380, 65528, 146829] avg=65536.0000 max=146829 max_rank=7 min=27170 min_rank=1 imbalance_ratio=2.240433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] avg=65536.0000 max=66195 max_rank=0 min=62930 min_rank=1 imbalance_ratio=1.010056 max_load_reduction=80634 max_load_reduction_pct=0.549169 moved_to_replicas=223966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[20095, 32967, 131017, 65523, 131042, 27, 13142, 130475] avg=65536.0000 max=131042 max_rank=4 min=27 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[20095, 32967, 131017, 65523, 131042, 27, 13142, 130475] avg=65536.0000 max=131042 max_rank=4 min=27 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[20095, 32967, 131017, 65523, 131042, 27, 13142, 130475] avg=65536.0000 max=131042 max_rank=4 min=27 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63424 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=387400 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[64390, 65514, 65300, 66192, 66192, 65877, 65591, 65232] avg=65536.0000 max=66192 max_rank=3 min=64390 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64855 max_load_reduction_pct=0.494899 moved_to_replicas=193899 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] avg=65536.0000 max=66195 max_rank=0 min=62930 min_rank=1 imbalance_ratio=1.010056 max_load_reduction=80634 max_load_reduction_pct=0.549169 moved_to_replicas=223966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[33227, 97727, 22, 131052, 32839, 65652, 130981, 32788] avg=65536.0000 max=131052 max_rank=3 min=22 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63424 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=387400 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63424 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=387400 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63424 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=387400 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[20095, 32967, 131017, 65523, 131042, 27, 13142, 130475] avg=65536.0000 max=131042 max_rank=4 min=27 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64762 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=321756 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[33227, 97727, 22, 131052, 32839, 65652, 130981, 32788] avg=65536.0000 max=131052 max_rank=3 min=22 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[10354, 71152, 65526, 98302, 86662, 126685, 32750, 32857] avg=65536.0000 max=126685 max_rank=5 min=10354 min_rank=0 imbalance_ratio=1.933060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[20095, 32967, 131017, 65523, 131042, 27, 13142, 130475] avg=65536.0000 max=131042 max_rank=4 min=27 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63424 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=387400 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[33227, 97727, 22, 131052, 32839, 65652, 130981, 32788] avg=65536.0000 max=131052 max_rank=3 min=22 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[33227, 97727, 22, 131052, 32839, 65652, 130981, 32788] avg=65536.0000 max=131052 max_rank=3 min=22 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[33227, 97727, 22, 131052, 32839, 65652, 130981, 32788] avg=65536.0000 max=131052 max_rank=3 min=22 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64762 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=321756 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[63587, 66192, 65526, 66192, 66190, 66193, 65441, 64967] avg=65536.0000 max=66193 max_rank=5 min=63587 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=60492 max_load_reduction_pct=0.477499 moved_to_replicas=230000 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63424 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=387400 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65584, 98291, 65552, 98333, 67912, 26, 95835, 32755] avg=65536.0000 max=98333 max_rank=3 min=26 min_rank=5 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64762 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=321756 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64762 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=321756 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64762 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=321756 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[33227, 97727, 22, 131052, 32839, 65652, 130981, 32788] avg=65536.0000 max=131052 max_rank=3 min=22 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65584, 98291, 65552, 98333, 67912, 26, 95835, 32755] avg=65536.0000 max=98333 max_rank=3 min=26 min_rank=5 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] avg=65536.0000 max=66192 max_rank=1 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=191173 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32787, 27170, 65521, 98288, 32785, 55380, 65528, 146829] avg=65536.0000 max=146829 max_rank=7 min=27170 min_rank=1 imbalance_ratio=2.240433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[33227, 97727, 22, 131052, 32839, 65652, 130981, 32788] avg=65536.0000 max=131052 max_rank=3 min=22 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64762 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=321756 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65584, 98291, 65552, 98333, 67912, 26, 95835, 32755] avg=65536.0000 max=98333 max_rank=3 min=26 min_rank=5 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65584, 98291, 65552, 98333, 67912, 26, 95835, 32755] avg=65536.0000 max=98333 max_rank=3 min=26 min_rank=5 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65584, 98291, 65552, 98333, 67912, 26, 95835, 32755] avg=65536.0000 max=98333 max_rank=3 min=26 min_rank=5 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] avg=65536.0000 max=66192 max_rank=1 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=191173 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[66195, 62930, 65521, 66192, 65541, 66191, 65528, 66190] avg=65536.0000 max=66195 max_rank=0 min=62930 min_rank=1 imbalance_ratio=1.010056 max_load_reduction=80634 max_load_reduction_pct=0.549169 moved_to_replicas=223966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64762 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=321756 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[98193, 98128, 163918, 32760, 142, 98360, 32743, 44] avg=65536.0000 max=163918 max_rank=2 min=44 min_rank=7 imbalance_ratio=2.501190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] avg=65536.0000 max=66192 max_rank=1 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=191173 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] avg=65536.0000 max=66192 max_rank=1 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=191173 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] avg=65536.0000 max=66192 max_rank=1 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=191173 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65584, 98291, 65552, 98333, 67912, 26, 95835, 32755] avg=65536.0000 max=98333 max_rank=3 min=26 min_rank=5 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[98193, 98128, 163918, 32760, 142, 98360, 32743, 44] avg=65536.0000 max=163918 max_rank=2 min=44 min_rank=7 imbalance_ratio=2.501190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] avg=65536.0000 max=66192 max_rank=0 min=64803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97726 max_load_reduction_pct=0.596188 moved_to_replicas=387762 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[20095, 32967, 131017, 65523, 131042, 27, 13142, 130475] avg=65536.0000 max=131042 max_rank=4 min=27 min_rank=5 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65584, 98291, 65552, 98333, 67912, 26, 95835, 32755] avg=65536.0000 max=98333 max_rank=3 min=26 min_rank=5 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] avg=65536.0000 max=66192 max_rank=1 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=191173 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[98193, 98128, 163918, 32760, 142, 98360, 32743, 44] avg=65536.0000 max=163918 max_rank=2 min=44 min_rank=7 imbalance_ratio=2.501190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[98193, 98128, 163918, 32760, 142, 98360, 32743, 44] avg=65536.0000 max=163918 max_rank=2 min=44 min_rank=7 imbalance_ratio=2.501190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[98193, 98128, 163918, 32760, 142, 98360, 32743, 44] avg=65536.0000 max=163918 max_rank=2 min=44 min_rank=7 imbalance_ratio=2.501190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] avg=65536.0000 max=66192 max_rank=0 min=64803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97726 max_load_reduction_pct=0.596188 moved_to_replicas=387762 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[63424, 65724, 66192, 65523, 66192, 64849, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=63424 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64850 max_load_reduction_pct=0.494880 moved_to_replicas=387400 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] avg=65536.0000 max=66192 max_rank=1 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=191173 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[32768, 65537, 70250, 65544, 98284, 71027, 32809, 88069] avg=65536.0000 max=98284 max_rank=4 min=32768 min_rank=0 imbalance_ratio=1.499695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] avg=65536.0000 max=66192 max_rank=0 min=64803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97726 max_load_reduction_pct=0.596188 moved_to_replicas=387762 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] avg=65536.0000 max=66192 max_rank=0 min=64803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97726 max_load_reduction_pct=0.596188 moved_to_replicas=387762 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] avg=65536.0000 max=66192 max_rank=0 min=64803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97726 max_load_reduction_pct=0.596188 moved_to_replicas=387762 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[98193, 98128, 163918, 32760, 142, 98360, 32743, 44] avg=65536.0000 max=163918 max_rank=2 min=44 min_rank=7 imbalance_ratio=2.501190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[32768, 65537, 70250, 65544, 98284, 71027, 32809, 88069] avg=65536.0000 max=98284 max_rank=4 min=32768 min_rank=0 imbalance_ratio=1.499695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] avg=65536.0000 max=66193 max_rank=5 min=63581 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32091 max_load_reduction_pct=0.326513 moved_to_replicas=125719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[33227, 97727, 22, 131052, 32839, 65652, 130981, 32788] avg=65536.0000 max=131052 max_rank=3 min=22 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] avg=65536.0000 max=66192 max_rank=0 min=64803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97726 max_load_reduction_pct=0.596188 moved_to_replicas=387762 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[98193, 98128, 163918, 32760, 142, 98360, 32743, 44] avg=65536.0000 max=163918 max_rank=2 min=44 min_rank=7 imbalance_ratio=2.501190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[32768, 65537, 70250, 65544, 98284, 71027, 32809, 88069] avg=65536.0000 max=98284 max_rank=4 min=32768 min_rank=0 imbalance_ratio=1.499695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[32768, 65537, 70250, 65544, 98284, 71027, 32809, 88069] avg=65536.0000 max=98284 max_rank=4 min=32768 min_rank=0 imbalance_ratio=1.499695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] avg=65536.0000 max=66193 max_rank=5 min=63581 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32091 max_load_reduction_pct=0.326513 moved_to_replicas=125719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[32768, 65537, 70250, 65544, 98284, 71027, 32809, 88069] avg=65536.0000 max=98284 max_rank=4 min=32768 min_rank=0 imbalance_ratio=1.499695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[64762, 66192, 64882, 66192, 64870, 65652, 66192, 65546] avg=65536.0000 max=66192 max_rank=1 min=64762 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64860 max_load_reduction_pct=0.494918 moved_to_replicas=321756 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] avg=65536.0000 max=66192 max_rank=0 min=64803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97726 max_load_reduction_pct=0.596188 moved_to_replicas=387762 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] avg=65536.0000 max=66193 max_rank=5 min=63581 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32091 max_load_reduction_pct=0.326513 moved_to_replicas=125719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131042, 32810, 35, 98242, 32506, 34228, 162631] avg=65536.0000 max=162631 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.481552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] avg=65536.0000 max=66193 max_rank=5 min=63581 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32091 max_load_reduction_pct=0.326513 moved_to_replicas=125719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] avg=65536.0000 max=66193 max_rank=5 min=63581 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32091 max_load_reduction_pct=0.326513 moved_to_replicas=125719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[32768, 65537, 70250, 65544, 98284, 71027, 32809, 88069] avg=65536.0000 max=98284 max_rank=4 min=32768 min_rank=0 imbalance_ratio=1.499695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131042, 32810, 35, 98242, 32506, 34228, 162631] avg=65536.0000 max=162631 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.481552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63804 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96439 max_load_reduction_pct=0.592993 moved_to_replicas=387166 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] avg=65536.0000 max=66193 max_rank=5 min=63581 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32091 max_load_reduction_pct=0.326513 moved_to_replicas=125719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65584, 98291, 65552, 98333, 67912, 26, 95835, 32755] avg=65536.0000 max=98333 max_rank=3 min=26 min_rank=5 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[32768, 65537, 70250, 65544, 98284, 71027, 32809, 88069] avg=65536.0000 max=98284 max_rank=4 min=32768 min_rank=0 imbalance_ratio=1.499695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131042, 32810, 35, 98242, 32506, 34228, 162631] avg=65536.0000 max=162631 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.481552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63804 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96439 max_load_reduction_pct=0.592993 moved_to_replicas=387166 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131042, 32810, 35, 98242, 32506, 34228, 162631] avg=65536.0000 max=162631 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.481552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131042, 32810, 35, 98242, 32506, 34228, 162631] avg=65536.0000 max=162631 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.481552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65584, 66192, 65552, 66192, 66192, 64266, 66192, 64118] avg=65536.0000 max=66192 max_rank=1 min=64118 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=191173 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[98242, 32815, 130965, 65556, 66, 32820, 65840, 97984] avg=65536.0000 max=130965 max_rank=2 min=66 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63804 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96439 max_load_reduction_pct=0.592993 moved_to_replicas=387166 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] avg=65536.0000 max=66193 max_rank=5 min=63581 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32091 max_load_reduction_pct=0.326513 moved_to_replicas=125719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63804 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96439 max_load_reduction_pct=0.592993 moved_to_replicas=387166 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63804 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96439 max_load_reduction_pct=0.592993 moved_to_replicas=387166 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131042, 32810, 35, 98242, 32506, 34228, 162631] avg=65536.0000 max=162631 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.481552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[98242, 32815, 130965, 65556, 66, 32820, 65840, 97984] avg=65536.0000 max=130965 max_rank=2 min=66 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] avg=65536.0000 max=66192 max_rank=0 min=64614 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257313 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63804 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96439 max_load_reduction_pct=0.592993 moved_to_replicas=387166 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[98193, 98128, 163918, 32760, 142, 98360, 32743, 44] avg=65536.0000 max=163918 max_rank=2 min=44 min_rank=7 imbalance_ratio=2.501190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[98242, 32815, 130965, 65556, 66, 32820, 65840, 97984] avg=65536.0000 max=130965 max_rank=2 min=66 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] avg=65536.0000 max=66192 max_rank=0 min=64614 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257313 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131042, 32810, 35, 98242, 32506, 34228, 162631] avg=65536.0000 max=162631 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.481552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[98242, 32815, 130965, 65556, 66, 32820, 65840, 97984] avg=65536.0000 max=130965 max_rank=2 min=66 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[98242, 32815, 130965, 65556, 66, 32820, 65840, 97984] avg=65536.0000 max=130965 max_rank=2 min=66 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[66192, 66192, 66192, 64928, 64836, 66192, 64953, 64803] avg=65536.0000 max=66192 max_rank=0 min=64803 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97726 max_load_reduction_pct=0.596188 moved_to_replicas=387762 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[65531, 7561, 130647, 65537, 32802, 91154, 98282, 32774] avg=65536.0000 max=130647 max_rank=2 min=7561 min_rank=1 imbalance_ratio=1.993515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] avg=65536.0000 max=66192 max_rank=0 min=64614 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257313 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[98242, 32815, 130965, 65556, 66, 32820, 65840, 97984] avg=65536.0000 max=130965 max_rank=2 min=66 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63804 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96439 max_load_reduction_pct=0.592993 moved_to_replicas=387166 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] avg=65536.0000 max=66192 max_rank=0 min=64614 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257313 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] avg=65536.0000 max=66192 max_rank=0 min=64614 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257313 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[65531, 7561, 130647, 65537, 32802, 91154, 98282, 32774] avg=65536.0000 max=130647 max_rank=2 min=7561 min_rank=1 imbalance_ratio=1.993515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] avg=65536.0000 max=66192 max_rank=2 min=64475 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64455 max_load_reduction_pct=0.493352 moved_to_replicas=244642 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] avg=65536.0000 max=66192 max_rank=0 min=64614 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257313 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[32768, 65537, 70250, 65544, 98284, 71027, 32809, 88069] avg=65536.0000 max=98284 max_rank=4 min=32768 min_rank=0 imbalance_ratio=1.499695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[65531, 7561, 130647, 65537, 32802, 91154, 98282, 32774] avg=65536.0000 max=130647 max_rank=2 min=7561 min_rank=1 imbalance_ratio=1.993515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] avg=65536.0000 max=66192 max_rank=2 min=64475 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64455 max_load_reduction_pct=0.493352 moved_to_replicas=244642 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[98242, 32815, 130965, 65556, 66, 32820, 65840, 97984] avg=65536.0000 max=130965 max_rank=2 min=66 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[65531, 7561, 130647, 65537, 32802, 91154, 98282, 32774] avg=65536.0000 max=130647 max_rank=2 min=7561 min_rank=1 imbalance_ratio=1.993515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[65531, 7561, 130647, 65537, 32802, 91154, 98282, 32774] avg=65536.0000 max=130647 max_rank=2 min=7561 min_rank=1 imbalance_ratio=1.993515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32559, 32789, 131054, 140009, 65433, 65544, 32793, 24107] avg=65536.0000 max=140009 max_rank=3 min=24107 min_rank=7 imbalance_ratio=2.136368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[64860, 65537, 66192, 65544, 66192, 66193, 63581, 66189] avg=65536.0000 max=66193 max_rank=5 min=63581 min_rank=6 imbalance_ratio=1.010025 max_load_reduction=32091 max_load_reduction_pct=0.326513 moved_to_replicas=125719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] avg=65536.0000 max=66192 max_rank=2 min=64475 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64455 max_load_reduction_pct=0.493352 moved_to_replicas=244642 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[65531, 7561, 130647, 65537, 32802, 91154, 98282, 32774] avg=65536.0000 max=130647 max_rank=2 min=7561 min_rank=1 imbalance_ratio=1.993515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] avg=65536.0000 max=66192 max_rank=0 min=64614 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257313 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] avg=65536.0000 max=66192 max_rank=2 min=64475 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64455 max_load_reduction_pct=0.493352 moved_to_replicas=244642 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] avg=65536.0000 max=66192 max_rank=2 min=64475 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64455 max_load_reduction_pct=0.493352 moved_to_replicas=244642 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32559, 32789, 131054, 140009, 65433, 65544, 32793, 24107] avg=65536.0000 max=140009 max_rank=3 min=24107 min_rank=7 imbalance_ratio=2.136368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] avg=65536.0000 max=66193 max_rank=1 min=63875 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=73816 max_load_reduction_pct=0.527223 moved_to_replicas=275870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] avg=65536.0000 max=66192 max_rank=2 min=64475 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64455 max_load_reduction_pct=0.493352 moved_to_replicas=244642 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32559, 32789, 131054, 140009, 65433, 65544, 32793, 24107] avg=65536.0000 max=140009 max_rank=3 min=24107 min_rank=7 imbalance_ratio=2.136368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[32794, 131042, 32810, 35, 98242, 32506, 34228, 162631] avg=65536.0000 max=162631 max_rank=7 min=35 min_rank=3 imbalance_ratio=2.481552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] avg=65536.0000 max=66193 max_rank=1 min=63875 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=73816 max_load_reduction_pct=0.527223 moved_to_replicas=275870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[65531, 7561, 130647, 65537, 32802, 91154, 98282, 32774] avg=65536.0000 max=130647 max_rank=2 min=7561 min_rank=1 imbalance_ratio=1.993515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32559, 32789, 131054, 140009, 65433, 65544, 32793, 24107] avg=65536.0000 max=140009 max_rank=3 min=24107 min_rank=7 imbalance_ratio=2.136368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32559, 32789, 131054, 140009, 65433, 65544, 32793, 24107] avg=65536.0000 max=140009 max_rank=3 min=24107 min_rank=7 imbalance_ratio=2.136368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] avg=65536.0000 max=66193 max_rank=1 min=63875 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=73816 max_load_reduction_pct=0.527223 moved_to_replicas=275870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[65549, 66192, 64905, 63804, 66192, 65262, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63804 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=96439 max_load_reduction_pct=0.592993 moved_to_replicas=387166 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32559, 32789, 131054, 140009, 65433, 65544, 32793, 24107] avg=65536.0000 max=140009 max_rank=3 min=24107 min_rank=7 imbalance_ratio=2.136368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] avg=65536.0000 max=66192 max_rank=2 min=64475 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64455 max_load_reduction_pct=0.493352 moved_to_replicas=244642 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] avg=65536.0000 max=66193 max_rank=1 min=63875 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=73816 max_load_reduction_pct=0.527223 moved_to_replicas=275870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] avg=65536.0000 max=66193 max_rank=1 min=63875 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=73816 max_load_reduction_pct=0.527223 moved_to_replicas=275870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] avg=65536.0000 max=66193 max_rank=1 min=63875 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=73816 max_load_reduction_pct=0.527223 moved_to_replicas=275870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[98242, 32815, 130965, 65556, 66, 32820, 65840, 97984] avg=65536.0000 max=130965 max_rank=2 min=66 min_rank=4 imbalance_ratio=1.998367 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32559, 32789, 131054, 140009, 65433, 65544, 32793, 24107] avg=65536.0000 max=140009 max_rank=3 min=24107 min_rank=7 imbalance_ratio=2.136368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[66192, 64832, 66192, 65556, 64614, 64870, 65840, 66192] avg=65536.0000 max=66192 max_rank=0 min=64614 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64773 max_load_reduction_pct=0.494583 moved_to_replicas=257313 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] avg=65536.0000 max=66193 max_rank=1 min=63875 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=73816 max_load_reduction_pct=0.527223 moved_to_replicas=275870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[65531, 7561, 130647, 65537, 32802, 91154, 98282, 32774] avg=65536.0000 max=130647 max_rank=2 min=7561 min_rank=1 imbalance_ratio=1.993515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[65531, 65279, 66192, 65537, 64892, 66190, 66192, 64475] avg=65536.0000 max=66192 max_rank=2 min=64475 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64455 max_load_reduction_pct=0.493352 moved_to_replicas=244642 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32559, 32789, 131054, 140009, 65433, 65544, 32793, 24107] avg=65536.0000 max=140009 max_rank=3 min=24107 min_rank=7 imbalance_ratio=2.136368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=2098669 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=504576 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[65313, 66193, 66192, 66191, 65433, 65544, 65547, 63875] avg=65536.0000 max=66193 max_rank=1 min=63875 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=73816 max_load_reduction_pct=0.527223 moved_to_replicas=275870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=2091655 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508800 peak_demand=66195 required_capacity=1.010056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=2091270 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509440 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=2089089 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=511872 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=2098602 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=502656 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=2100968 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=502400 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=2102052 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=499072 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=2104911 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=497408 peak_demand=66194 required_capacity=1.010040 + [2026-08-30 05:14:11] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 542.4 | throughput per GPU (TFLOP/s/GPU): 614.6 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.038034E+01 | loss scale: 1.0 | grad norm: 32.705 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=81920 expected_recv=125114 expected_overflow=0 exact_recv_per_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132307 expected_overflow=0 exact_recv_per_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=81920 expected_recv=129234 expected_overflow=0 exact_recv_per_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=81920 expected_recv=125114 actual_valid_recv=126848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132307 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=126848 slack_slots=36992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=81920 expected_recv=129234 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=81920 expected_recv=128629 expected_overflow=0 exact_recv_per_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=81920 expected_recv=129546 expected_overflow=0 exact_recv_per_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=81920 expected_recv=129704 expected_overflow=0 exact_recv_per_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=81920 expected_recv=128629 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=81920 expected_recv=132388 expected_overflow=0 exact_recv_per_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=81920 expected_recv=129546 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=81920 expected_recv=129704 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=81920 expected_recv=132388 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131157 expected_overflow=0 exact_recv_per_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=134144 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131157 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131082 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=81920 expected_recv=127078 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=81920 expected_recv=129805 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131082 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=81920 expected_recv=127078 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131075 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=134272 slack_slots=29568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=81920 expected_recv=129805 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=128640 slack_slots=35200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131075 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=81920 expected_recv=129982 expected_overflow=0 exact_recv_per_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130358 expected_overflow=0 exact_recv_per_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=81920 expected_recv=129982 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130914 expected_overflow=0 exact_recv_per_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130358 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=81920 expected_recv=129900 expected_overflow=0 exact_recv_per_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130914 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=81920 expected_recv=129900 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130270 expected_overflow=0 exact_recv_per_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130270 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131077 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=81920 expected_recv=128474 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131077 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=81920 expected_recv=128474 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=81920 expected_recv=129688 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=81920 expected_recv=129799 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=129792 slack_slots=34048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=81920 expected_recv=129688 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=81920 expected_recv=129799 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=81920 expected_recv=129414 expected_overflow=0 exact_recv_per_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=81920 expected_recv=129414 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131125 expected_overflow=0 exact_recv_per_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131125 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=81920 expected_recv=129962 expected_overflow=0 exact_recv_per_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=81920 expected_recv=129814 expected_overflow=0 exact_recv_per_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131112 expected_overflow=0 exact_recv_per_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=81920 expected_recv=129814 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=81920 expected_recv=129962 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131034 expected_overflow=0 exact_recv_per_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131112 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131034 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131733 expected_overflow=0 exact_recv_per_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131733 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=81920 expected_recv=130481 expected_overflow=0 exact_recv_per_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=81920 expected_recv=130481 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=81920 expected_recv=130029 expected_overflow=0 exact_recv_per_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131250 expected_overflow=0 exact_recv_per_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=81920 expected_recv=130029 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=81920 expected_recv=125386 expected_overflow=0 exact_recv_per_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=81920 expected_recv=129282 expected_overflow=0 exact_recv_per_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131250 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=81920 expected_recv=125386 actual_valid_recv=126592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=81920 expected_recv=129282 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131460 expected_overflow=0 exact_recv_per_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=126592 slack_slots=37248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131460 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131124 expected_overflow=0 exact_recv_per_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=81920 expected_recv=124689 expected_overflow=0 exact_recv_per_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131124 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=81920 expected_recv=124689 actual_valid_recv=126080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=126080 slack_slots=37760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131070 expected_overflow=0 exact_recv_per_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=81920 expected_recv=130897 expected_overflow=0 exact_recv_per_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131070 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=81920 expected_recv=130897 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=81920 expected_recv=129902 expected_overflow=0 exact_recv_per_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=81920 expected_recv=129902 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131070 expected_overflow=0 exact_recv_per_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131070 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=81920 expected_recv=129931 expected_overflow=0 exact_recv_per_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=81920 expected_recv=129931 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131035 expected_overflow=0 exact_recv_per_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130896 expected_overflow=0 exact_recv_per_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131035 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131076 expected_overflow=0 exact_recv_per_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130896 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131076 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131054 expected_overflow=0 exact_recv_per_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131054 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130990 expected_overflow=0 exact_recv_per_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131167 expected_overflow=0 exact_recv_per_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131072 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130990 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131167 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=81920 expected_recv=129699 expected_overflow=0 exact_recv_per_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=81920 expected_recv=129699 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=81920 expected_recv=129773 expected_overflow=0 exact_recv_per_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=81920 expected_recv=129766 expected_overflow=0 exact_recv_per_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=81920 expected_recv=129773 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=81920 expected_recv=129766 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131021 expected_overflow=0 exact_recv_per_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=81920 expected_recv=129867 expected_overflow=0 exact_recv_per_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131021 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=81920 expected_recv=129867 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131132 expected_overflow=0 exact_recv_per_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131132 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131091 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131112 expected_overflow=0 exact_recv_per_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131091 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131112 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=81920 expected_recv=129547 expected_overflow=0 exact_recv_per_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131106 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=81920 expected_recv=129547 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131106 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=81920 expected_recv=129844 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=81920 expected_recv=129627 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=81920 expected_recv=129844 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=81920 expected_recv=129627 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131164 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=81920 expected_recv=125653 expected_overflow=0 exact_recv_per_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131164 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=81920 expected_recv=130976 expected_overflow=0 exact_recv_per_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=81920 expected_recv=125653 actual_valid_recv=126720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=81920 expected_recv=130976 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=126720 slack_slots=37120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=81920 expected_recv=129546 expected_overflow=0 exact_recv_per_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131066 expected_overflow=0 exact_recv_per_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=81920 expected_recv=129546 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131169 expected_overflow=0 exact_recv_per_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131066 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=134144 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131169 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=81920 expected_recv=130305 expected_overflow=0 exact_recv_per_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131152 expected_overflow=0 exact_recv_per_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=81920 expected_recv=130305 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131152 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=81920 expected_recv=129669 expected_overflow=0 exact_recv_per_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=81920 expected_recv=129863 expected_overflow=0 exact_recv_per_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=81920 expected_recv=129669 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=81920 expected_recv=129863 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131118 expected_overflow=0 exact_recv_per_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131118 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131174 expected_overflow=0 exact_recv_per_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=81920 expected_recv=130786 expected_overflow=0 exact_recv_per_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131174 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=81920 expected_recv=130786 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131083 expected_overflow=0 exact_recv_per_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131083 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=81920 expected_recv=128271 expected_overflow=0 exact_recv_per_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[17492, 89537, 87157, 44850, 77358, 66743, 77316, 63835] avg=65536.0000 max=89537 max_rank=1 min=17492 min_rank=0 imbalance_ratio=1.366226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=81920 expected_recv=128271 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] avg=65536.0000 max=66193 max_rank=1 min=63300 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23344 max_load_reduction_pct=0.260719 moved_to_replicas=133204 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[17492, 89537, 87157, 44850, 77358, 66743, 77316, 63835] avg=65536.0000 max=89537 max_rank=1 min=17492 min_rank=0 imbalance_ratio=1.366226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131034 expected_overflow=0 exact_recv_per_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[17492, 89537, 87157, 44850, 77358, 66743, 77316, 63835] avg=65536.0000 max=89537 max_rank=1 min=17492 min_rank=0 imbalance_ratio=1.366226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131034 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[17492, 89537, 87157, 44850, 77358, 66743, 77316, 63835] avg=65536.0000 max=89537 max_rank=1 min=17492 min_rank=0 imbalance_ratio=1.366226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[17492, 89537, 87157, 44850, 77358, 66743, 77316, 63835] avg=65536.0000 max=89537 max_rank=1 min=17492 min_rank=0 imbalance_ratio=1.366226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] avg=65536.0000 max=66193 max_rank=1 min=63300 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23344 max_load_reduction_pct=0.260719 moved_to_replicas=133204 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[17492, 89537, 87157, 44850, 77358, 66743, 77316, 63835] avg=65536.0000 max=89537 max_rank=1 min=17492 min_rank=0 imbalance_ratio=1.366226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[100551, 57707, 65030, 68715, 99633, 63755, 36118, 32779] avg=65536.0000 max=100551 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.534286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[17492, 89537, 87157, 44850, 77358, 66743, 77316, 63835] avg=65536.0000 max=89537 max_rank=1 min=17492 min_rank=0 imbalance_ratio=1.366226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] avg=65536.0000 max=66193 max_rank=1 min=63300 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23344 max_load_reduction_pct=0.260719 moved_to_replicas=133204 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] avg=65536.0000 max=66193 max_rank=1 min=63300 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23344 max_load_reduction_pct=0.260719 moved_to_replicas=133204 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] avg=65536.0000 max=66193 max_rank=1 min=63300 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23344 max_load_reduction_pct=0.260719 moved_to_replicas=133204 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] avg=65536.0000 max=66194 max_rank=6 min=63755 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34357 max_load_reduction_pct=0.341687 moved_to_replicas=142395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[17492, 89537, 87157, 44850, 77358, 66743, 77316, 63835] avg=65536.0000 max=89537 max_rank=1 min=17492 min_rank=0 imbalance_ratio=1.366226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] avg=65536.0000 max=66193 max_rank=1 min=63300 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23344 max_load_reduction_pct=0.260719 moved_to_replicas=133204 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] avg=65536.0000 max=66193 max_rank=1 min=63300 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23344 max_load_reduction_pct=0.260719 moved_to_replicas=133204 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[100551, 57707, 65030, 68715, 99633, 63755, 36118, 32779] avg=65536.0000 max=100551 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.534286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[100551, 57707, 65030, 68715, 99633, 63755, 36118, 32779] avg=65536.0000 max=100551 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.534286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[100551, 57707, 65030, 68715, 99633, 63755, 36118, 32779] avg=65536.0000 max=100551 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.534286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[100551, 57707, 65030, 68715, 99633, 63755, 36118, 32779] avg=65536.0000 max=100551 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.534286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[63300, 66193, 66192, 66192, 66192, 66192, 66192, 63835] avg=65536.0000 max=66193 max_rank=1 min=63300 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23344 max_load_reduction_pct=0.260719 moved_to_replicas=133204 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] avg=65536.0000 max=66194 max_rank=6 min=63755 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34357 max_load_reduction_pct=0.341687 moved_to_replicas=142395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65534, 23588, 59689, 33083, 98295, 80427, 65541, 98131] avg=65536.0000 max=98295 max_rank=4 min=23588 min_rank=1 imbalance_ratio=1.499863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] avg=65536.0000 max=66194 max_rank=6 min=63755 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34357 max_load_reduction_pct=0.341687 moved_to_replicas=142395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[100551, 57707, 65030, 68715, 99633, 63755, 36118, 32779] avg=65536.0000 max=100551 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.534286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[100551, 57707, 65030, 68715, 99633, 63755, 36118, 32779] avg=65536.0000 max=100551 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.534286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] avg=65536.0000 max=66194 max_rank=6 min=63755 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34357 max_load_reduction_pct=0.341687 moved_to_replicas=142395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] avg=65536.0000 max=66194 max_rank=6 min=63755 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34357 max_load_reduction_pct=0.341687 moved_to_replicas=142395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] avg=65536.0000 max=66193 max_rank=5 min=63423 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32102 max_load_reduction_pct=0.326588 moved_to_replicas=157622 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65534, 23588, 59689, 33083, 98295, 80427, 65541, 98131] avg=65536.0000 max=98295 max_rank=4 min=23588 min_rank=1 imbalance_ratio=1.499863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[100551, 57707, 65030, 68715, 99633, 63755, 36118, 32779] avg=65536.0000 max=100551 max_rank=0 min=32779 min_rank=7 imbalance_ratio=1.534286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] avg=65536.0000 max=66194 max_rank=6 min=63755 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34357 max_load_reduction_pct=0.341687 moved_to_replicas=142395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] avg=65536.0000 max=66194 max_rank=6 min=63755 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34357 max_load_reduction_pct=0.341687 moved_to_replicas=142395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65534, 23588, 59689, 33083, 98295, 80427, 65541, 98131] avg=65536.0000 max=98295 max_rank=4 min=23588 min_rank=1 imbalance_ratio=1.499863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65534, 23588, 59689, 33083, 98295, 80427, 65541, 98131] avg=65536.0000 max=98295 max_rank=4 min=23588 min_rank=1 imbalance_ratio=1.499863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65534, 23588, 59689, 33083, 98295, 80427, 65541, 98131] avg=65536.0000 max=98295 max_rank=4 min=23588 min_rank=1 imbalance_ratio=1.499863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] avg=65536.0000 max=66193 max_rank=5 min=63423 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32102 max_load_reduction_pct=0.326588 moved_to_replicas=157622 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65497, 32842, 98238, 98502, 163802, 65355] avg=65536.0000 max=163802 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[66192, 65189, 65030, 66192, 66192, 63755, 66194, 65544] avg=65536.0000 max=66194 max_rank=6 min=63755 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34357 max_load_reduction_pct=0.341687 moved_to_replicas=142395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] avg=65536.0000 max=66193 max_rank=5 min=63423 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32102 max_load_reduction_pct=0.326588 moved_to_replicas=157622 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65534, 23588, 59689, 33083, 98295, 80427, 65541, 98131] avg=65536.0000 max=98295 max_rank=4 min=23588 min_rank=1 imbalance_ratio=1.499863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65534, 23588, 59689, 33083, 98295, 80427, 65541, 98131] avg=65536.0000 max=98295 max_rank=4 min=23588 min_rank=1 imbalance_ratio=1.499863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] avg=65536.0000 max=66193 max_rank=5 min=63423 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32102 max_load_reduction_pct=0.326588 moved_to_replicas=157622 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] avg=65536.0000 max=66193 max_rank=5 min=63423 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32102 max_load_reduction_pct=0.326588 moved_to_replicas=157622 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] avg=65536.0000 max=66192 max_rank=4 min=64866 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97610 max_load_reduction_pct=0.595902 moved_to_replicas=324141 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] avg=65536.0000 max=66193 max_rank=5 min=63423 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32102 max_load_reduction_pct=0.326588 moved_to_replicas=157622 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65497, 32842, 98238, 98502, 163802, 65355] avg=65536.0000 max=163802 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65534, 23588, 59689, 33083, 98295, 80427, 65541, 98131] avg=65536.0000 max=98295 max_rank=4 min=23588 min_rank=1 imbalance_ratio=1.499863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] avg=65536.0000 max=66193 max_rank=5 min=63423 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32102 max_load_reduction_pct=0.326588 moved_to_replicas=157622 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65497, 32842, 98238, 98502, 163802, 65355] avg=65536.0000 max=163802 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65497, 32842, 98238, 98502, 163802, 65355] avg=65536.0000 max=163802 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65497, 32842, 98238, 98502, 163802, 65355] avg=65536.0000 max=163802 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[95097, 65539, 52718, 98279, 98307, 48789, 32787, 32772] avg=65536.0000 max=98307 max_rank=4 min=32772 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] avg=65536.0000 max=66192 max_rank=4 min=64866 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97610 max_load_reduction_pct=0.595902 moved_to_replicas=324141 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65534, 66191, 63423, 65022, 66192, 66193, 65541, 66192] avg=65536.0000 max=66193 max_rank=5 min=63423 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=32102 max_load_reduction_pct=0.326588 moved_to_replicas=157622 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] avg=65536.0000 max=66192 max_rank=4 min=64866 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97610 max_load_reduction_pct=0.595902 moved_to_replicas=324141 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65497, 32842, 98238, 98502, 163802, 65355] avg=65536.0000 max=163802 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] avg=65536.0000 max=66192 max_rank=4 min=64866 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97610 max_load_reduction_pct=0.595902 moved_to_replicas=324141 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] avg=65536.0000 max=66192 max_rank=4 min=64866 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97610 max_load_reduction_pct=0.595902 moved_to_replicas=324141 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=187338 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65497, 32842, 98238, 98502, 163802, 65355] avg=65536.0000 max=163802 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] avg=65536.0000 max=66192 max_rank=4 min=64866 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97610 max_load_reduction_pct=0.595902 moved_to_replicas=324141 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[95097, 65539, 52718, 98279, 98307, 48789, 32787, 32772] avg=65536.0000 max=98307 max_rank=4 min=32772 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[34, 18, 65497, 32842, 98238, 98502, 163802, 65355] avg=65536.0000 max=163802 max_rank=6 min=18 min_rank=1 imbalance_ratio=2.499420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[95097, 65539, 52718, 98279, 98307, 48789, 32787, 32772] avg=65536.0000 max=98307 max_rank=4 min=32772 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[95097, 65539, 52718, 98279, 98307, 48789, 32787, 32772] avg=65536.0000 max=98307 max_rank=4 min=32772 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] avg=65536.0000 max=66192 max_rank=4 min=64866 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97610 max_load_reduction_pct=0.595902 moved_to_replicas=324141 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[95097, 65539, 52718, 98279, 98307, 48789, 32787, 32772] avg=65536.0000 max=98307 max_rank=4 min=32772 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 93876, 70342, 65543, 98386, 32791, 32878] avg=65536.0000 max=98386 max_rank=5 min=32791 min_rank=6 imbalance_ratio=1.501251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=187338 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65106, 64866, 65497, 64888, 66192, 66192, 66192, 65355] avg=65536.0000 max=66192 max_rank=4 min=64866 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97610 max_load_reduction_pct=0.595902 moved_to_replicas=324141 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=187338 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[95097, 65539, 52718, 98279, 98307, 48789, 32787, 32772] avg=65536.0000 max=98307 max_rank=4 min=32772 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=187338 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=187338 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] avg=65536.0000 max=66192 max_rank=5 min=64715 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32194 max_load_reduction_pct=0.327221 moved_to_replicas=128176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[95097, 65539, 52718, 98279, 98307, 48789, 32787, 32772] avg=65536.0000 max=98307 max_rank=4 min=32772 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 93876, 70342, 65543, 98386, 32791, 32878] avg=65536.0000 max=98386 max_rank=5 min=32791 min_rank=6 imbalance_ratio=1.501251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=187338 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[95097, 65539, 52718, 98279, 98307, 48789, 32787, 32772] avg=65536.0000 max=98307 max_rank=4 min=32772 min_rank=7 imbalance_ratio=1.500046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 93876, 70342, 65543, 98386, 32791, 32878] avg=65536.0000 max=98386 max_rank=5 min=32791 min_rank=6 imbalance_ratio=1.501251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 93876, 70342, 65543, 98386, 32791, 32878] avg=65536.0000 max=98386 max_rank=5 min=32791 min_rank=6 imbalance_ratio=1.501251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 93876, 70342, 65543, 98386, 32791, 32878] avg=65536.0000 max=98386 max_rank=5 min=32791 min_rank=6 imbalance_ratio=1.501251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=187338 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32782, 65329, 9520, 32783, 145385, 65578, 107377] avg=65536.0000 max=145385 max_rank=5 min=9520 min_rank=3 imbalance_ratio=2.218399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] avg=65536.0000 max=66192 max_rank=5 min=64715 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32194 max_load_reduction_pct=0.327221 moved_to_replicas=128176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[66192, 65539, 64218, 66192, 66192, 66194, 64874, 64887] avg=65536.0000 max=66194 max_rank=5 min=64218 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32113 max_load_reduction_pct=0.326660 moved_to_replicas=187338 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] avg=65536.0000 max=66192 max_rank=5 min=64715 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32194 max_load_reduction_pct=0.327221 moved_to_replicas=128176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] avg=65536.0000 max=66192 max_rank=5 min=64715 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32194 max_load_reduction_pct=0.327221 moved_to_replicas=128176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 93876, 70342, 65543, 98386, 32791, 32878] avg=65536.0000 max=98386 max_rank=5 min=32791 min_rank=6 imbalance_ratio=1.501251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] avg=65536.0000 max=66192 max_rank=5 min=64715 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32194 max_load_reduction_pct=0.327221 moved_to_replicas=128176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] avg=65536.0000 max=66194 max_rank=7 min=63730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79191 max_load_reduction_pct=0.544699 moved_to_replicas=237023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 93876, 70342, 65543, 98386, 32791, 32878] avg=65536.0000 max=98386 max_rank=5 min=32791 min_rank=6 imbalance_ratio=1.501251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] avg=65536.0000 max=66192 max_rank=5 min=64715 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32194 max_load_reduction_pct=0.327221 moved_to_replicas=128176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32782, 65329, 9520, 32783, 145385, 65578, 107377] avg=65536.0000 max=145385 max_rank=5 min=9520 min_rank=3 imbalance_ratio=2.218399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 93876, 70342, 65543, 98386, 32791, 32878] avg=65536.0000 max=98386 max_rank=5 min=32791 min_rank=6 imbalance_ratio=1.501251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32782, 65329, 9520, 32783, 145385, 65578, 107377] avg=65536.0000 max=145385 max_rank=5 min=9520 min_rank=3 imbalance_ratio=2.218399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32782, 65329, 9520, 32783, 145385, 65578, 107377] avg=65536.0000 max=145385 max_rank=5 min=9520 min_rank=3 imbalance_ratio=2.218399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32782, 65329, 9520, 32783, 145385, 65578, 107377] avg=65536.0000 max=145385 max_rank=5 min=9520 min_rank=3 imbalance_ratio=2.218399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[61814, 32936, 143838, 65562, 56290, 16, 32777, 131055] avg=65536.0000 max=143838 max_rank=2 min=16 min_rank=5 imbalance_ratio=2.194794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] avg=65536.0000 max=66192 max_rank=5 min=64715 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32194 max_load_reduction_pct=0.327221 moved_to_replicas=128176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] avg=65536.0000 max=66194 max_rank=7 min=63730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79191 max_load_reduction_pct=0.544699 moved_to_replicas=237023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[64911, 65561, 66191, 66190, 65543, 66192, 64985, 64715] avg=65536.0000 max=66192 max_rank=5 min=64715 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32194 max_load_reduction_pct=0.327221 moved_to_replicas=128176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] avg=65536.0000 max=66194 max_rank=7 min=63730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79191 max_load_reduction_pct=0.544699 moved_to_replicas=237023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32782, 65329, 9520, 32783, 145385, 65578, 107377] avg=65536.0000 max=145385 max_rank=5 min=9520 min_rank=3 imbalance_ratio=2.218399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] avg=65536.0000 max=66194 max_rank=7 min=63730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79191 max_load_reduction_pct=0.544699 moved_to_replicas=237023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] avg=65536.0000 max=66194 max_rank=7 min=63730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79191 max_load_reduction_pct=0.544699 moved_to_replicas=237023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62726 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77645 max_load_reduction_pct=0.539809 moved_to_replicas=285555 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32782, 65329, 9520, 32783, 145385, 65578, 107377] avg=65536.0000 max=145385 max_rank=5 min=9520 min_rank=3 imbalance_ratio=2.218399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] avg=65536.0000 max=66194 max_rank=7 min=63730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79191 max_load_reduction_pct=0.544699 moved_to_replicas=237023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[61814, 32936, 143838, 65562, 56290, 16, 32777, 131055] avg=65536.0000 max=143838 max_rank=2 min=16 min_rank=5 imbalance_ratio=2.194794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[65534, 32782, 65329, 9520, 32783, 145385, 65578, 107377] avg=65536.0000 max=145385 max_rank=5 min=9520 min_rank=3 imbalance_ratio=2.218399 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[61814, 32936, 143838, 65562, 56290, 16, 32777, 131055] avg=65536.0000 max=143838 max_rank=2 min=16 min_rank=5 imbalance_ratio=2.194794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[61814, 32936, 143838, 65562, 56290, 16, 32777, 131055] avg=65536.0000 max=143838 max_rank=2 min=16 min_rank=5 imbalance_ratio=2.194794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[61814, 32936, 143838, 65562, 56290, 16, 32777, 131055] avg=65536.0000 max=143838 max_rank=2 min=16 min_rank=5 imbalance_ratio=2.194794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[55853, 65482, 23, 98318, 65534, 95426, 39601, 104051] avg=65536.0000 max=104051 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.587692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] avg=65536.0000 max=66194 max_rank=7 min=63730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79191 max_load_reduction_pct=0.544699 moved_to_replicas=237023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62726 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77645 max_load_reduction_pct=0.539809 moved_to_replicas=285555 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[65534, 65542, 65329, 66190, 63730, 66191, 65578, 66194] avg=65536.0000 max=66194 max_rank=7 min=63730 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79191 max_load_reduction_pct=0.544699 moved_to_replicas=237023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62726 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77645 max_load_reduction_pct=0.539809 moved_to_replicas=285555 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62726 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77645 max_load_reduction_pct=0.539809 moved_to_replicas=285555 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[61814, 32936, 143838, 65562, 56290, 16, 32777, 131055] avg=65536.0000 max=143838 max_rank=2 min=16 min_rank=5 imbalance_ratio=2.194794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62726 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77645 max_load_reduction_pct=0.539809 moved_to_replicas=285555 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=62312 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37857 max_load_reduction_pct=0.363831 moved_to_replicas=197898 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[61814, 32936, 143838, 65562, 56290, 16, 32777, 131055] avg=65536.0000 max=143838 max_rank=2 min=16 min_rank=5 imbalance_ratio=2.194794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[55853, 65482, 23, 98318, 65534, 95426, 39601, 104051] avg=65536.0000 max=104051 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.587692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62726 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77645 max_load_reduction_pct=0.539809 moved_to_replicas=285555 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[55853, 65482, 23, 98318, 65534, 95426, 39601, 104051] avg=65536.0000 max=104051 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.587692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[61814, 32936, 143838, 65562, 56290, 16, 32777, 131055] avg=65536.0000 max=143838 max_rank=2 min=16 min_rank=5 imbalance_ratio=2.194794 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[55853, 65482, 23, 98318, 65534, 95426, 39601, 104051] avg=65536.0000 max=104051 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.587692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[55853, 65482, 23, 98318, 65534, 95426, 39601, 104051] avg=65536.0000 max=104051 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.587692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98367, 32792, 65453, 65537, 65530, 98300, 65517] avg=65536.0000 max=98367 max_rank=1 min=32792 min_rank=0 imbalance_ratio=1.500961 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62726 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77645 max_load_reduction_pct=0.539809 moved_to_replicas=285555 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=62312 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37857 max_load_reduction_pct=0.363831 moved_to_replicas=197898 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=62312 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37857 max_load_reduction_pct=0.363831 moved_to_replicas=197898 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[62726, 65697, 66193, 65562, 66192, 65535, 66191, 66192] avg=65536.0000 max=66193 max_rank=2 min=62726 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=77645 max_load_reduction_pct=0.539809 moved_to_replicas=285555 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=62312 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37857 max_load_reduction_pct=0.363831 moved_to_replicas=197898 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[55853, 65482, 23, 98318, 65534, 95426, 39601, 104051] avg=65536.0000 max=104051 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.587692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=62312 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37857 max_load_reduction_pct=0.363831 moved_to_replicas=197898 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] avg=65536.0000 max=66192 max_rank=1 min=64900 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32175 max_load_reduction_pct=0.327091 moved_to_replicas=128596 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[55853, 65482, 23, 98318, 65534, 95426, 39601, 104051] avg=65536.0000 max=104051 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.587692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98367, 32792, 65453, 65537, 65530, 98300, 65517] avg=65536.0000 max=98367 max_rank=1 min=32792 min_rank=0 imbalance_ratio=1.500961 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=62312 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37857 max_load_reduction_pct=0.363831 moved_to_replicas=197898 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98367, 32792, 65453, 65537, 65530, 98300, 65517] avg=65536.0000 max=98367 max_rank=1 min=32792 min_rank=0 imbalance_ratio=1.500961 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98367, 32792, 65453, 65537, 65530, 98300, 65517] avg=65536.0000 max=98367 max_rank=1 min=32792 min_rank=0 imbalance_ratio=1.500961 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[55853, 65482, 23, 98318, 65534, 95426, 39601, 104051] avg=65536.0000 max=104051 max_rank=7 min=23 min_rank=2 imbalance_ratio=1.587692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98367, 32792, 65453, 65537, 65530, 98300, 65517] avg=65536.0000 max=98367 max_rank=1 min=32792 min_rank=0 imbalance_ratio=1.500961 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32775, 65577, 32876, 65518, 131041, 98161, 32781] avg=65536.0000 max=131041 max_rank=5 min=32775 min_rank=1 imbalance_ratio=1.999527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=62312 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37857 max_load_reduction_pct=0.363831 moved_to_replicas=197898 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] avg=65536.0000 max=66192 max_rank=1 min=64900 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32175 max_load_reduction_pct=0.327091 moved_to_replicas=128596 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] avg=65536.0000 max=66192 max_rank=1 min=64900 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32175 max_load_reduction_pct=0.327091 moved_to_replicas=128596 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] avg=65536.0000 max=66192 max_rank=1 min=64900 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32175 max_load_reduction_pct=0.327091 moved_to_replicas=128596 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[62312, 65482, 66190, 66192, 65534, 66192, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=62312 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37857 max_load_reduction_pct=0.363831 moved_to_replicas=197898 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98367, 32792, 65453, 65537, 65530, 98300, 65517] avg=65536.0000 max=98367 max_rank=1 min=32792 min_rank=0 imbalance_ratio=1.500961 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] avg=65536.0000 max=66192 max_rank=1 min=64900 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32175 max_load_reduction_pct=0.327091 moved_to_replicas=128596 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] avg=65536.0000 max=66192 max_rank=5 min=64845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64849 max_load_reduction_pct=0.494876 moved_to_replicas=193349 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98367, 32792, 65453, 65537, 65530, 98300, 65517] avg=65536.0000 max=98367 max_rank=1 min=32792 min_rank=0 imbalance_ratio=1.500961 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32775, 65577, 32876, 65518, 131041, 98161, 32781] avg=65536.0000 max=131041 max_rank=5 min=32775 min_rank=1 imbalance_ratio=1.999527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] avg=65536.0000 max=66192 max_rank=1 min=64900 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32175 max_load_reduction_pct=0.327091 moved_to_replicas=128596 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32775, 65577, 32876, 65518, 131041, 98161, 32781] avg=65536.0000 max=131041 max_rank=5 min=32775 min_rank=1 imbalance_ratio=1.999527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32775, 65577, 32876, 65518, 131041, 98161, 32781] avg=65536.0000 max=131041 max_rank=5 min=32775 min_rank=1 imbalance_ratio=1.999527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32792, 98367, 32792, 65453, 65537, 65530, 98300, 65517] avg=65536.0000 max=98367 max_rank=1 min=32792 min_rank=0 imbalance_ratio=1.500961 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32775, 65577, 32876, 65518, 131041, 98161, 32781] avg=65536.0000 max=131041 max_rank=5 min=32775 min_rank=1 imbalance_ratio=1.999527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[24, 32821, 98268, 98291, 131032, 32751, 65550, 65551] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] avg=65536.0000 max=66192 max_rank=1 min=64900 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32175 max_load_reduction_pct=0.327091 moved_to_replicas=128596 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] avg=65536.0000 max=66192 max_rank=5 min=64845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64849 max_load_reduction_pct=0.494876 moved_to_replicas=193349 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] avg=65536.0000 max=66192 max_rank=5 min=64845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64849 max_load_reduction_pct=0.494876 moved_to_replicas=193349 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] avg=65536.0000 max=66192 max_rank=5 min=64845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64849 max_load_reduction_pct=0.494876 moved_to_replicas=193349 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[64967, 66192, 64900, 65453, 65537, 65530, 66192, 65517] avg=65536.0000 max=66192 max_rank=1 min=64900 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32175 max_load_reduction_pct=0.327091 moved_to_replicas=128596 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] avg=65536.0000 max=66192 max_rank=5 min=64845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64849 max_load_reduction_pct=0.494876 moved_to_replicas=193349 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32775, 65577, 32876, 65518, 131041, 98161, 32781] avg=65536.0000 max=131041 max_rank=5 min=32775 min_rank=1 imbalance_ratio=1.999527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] avg=65536.0000 max=66192 max_rank=2 min=64831 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32775, 65577, 32876, 65518, 131041, 98161, 32781] avg=65536.0000 max=131041 max_rank=5 min=32775 min_rank=1 imbalance_ratio=1.999527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[24, 32821, 98268, 98291, 131032, 32751, 65550, 65551] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[24, 32821, 98268, 98291, 131032, 32751, 65550, 65551] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[24, 32821, 98268, 98291, 131032, 32751, 65550, 65551] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[65559, 32775, 65577, 32876, 65518, 131041, 98161, 32781] avg=65536.0000 max=131041 max_rank=5 min=32775 min_rank=1 imbalance_ratio=1.999527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] avg=65536.0000 max=66192 max_rank=5 min=64845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64849 max_load_reduction_pct=0.494876 moved_to_replicas=193349 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[24, 32821, 98268, 98291, 131032, 32751, 65550, 65551] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] avg=65536.0000 max=66192 max_rank=5 min=64845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64849 max_load_reduction_pct=0.494876 moved_to_replicas=193349 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[32784, 98287, 65553, 32802, 65558, 65513, 28, 163763] avg=65536.0000 max=163763 max_rank=7 min=28 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] avg=65536.0000 max=66192 max_rank=2 min=64831 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] avg=65536.0000 max=66192 max_rank=2 min=64831 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] avg=65536.0000 max=66192 max_rank=2 min=64831 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[65559, 65535, 65577, 64845, 65518, 66192, 66192, 64870] avg=65536.0000 max=66192 max_rank=5 min=64845 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64849 max_load_reduction_pct=0.494876 moved_to_replicas=193349 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] avg=65536.0000 max=66192 max_rank=2 min=64831 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] avg=65536.0000 max=66192 max_rank=1 min=64841 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[24, 32821, 98268, 98291, 131032, 32751, 65550, 65551] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[24, 32821, 98268, 98291, 131032, 32751, 65550, 65551] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[32784, 98287, 65553, 32802, 65558, 65513, 28, 163763] avg=65536.0000 max=163763 max_rank=7 min=28 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[32784, 98287, 65553, 32802, 65558, 65513, 28, 163763] avg=65536.0000 max=163763 max_rank=7 min=28 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[32784, 98287, 65553, 32802, 65558, 65513, 28, 163763] avg=65536.0000 max=163763 max_rank=7 min=28 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[24, 32821, 98268, 98291, 131032, 32751, 65550, 65551] avg=65536.0000 max=131032 max_rank=4 min=24 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[32784, 98287, 65553, 32802, 65558, 65513, 28, 163763] avg=65536.0000 max=163763 max_rank=7 min=28 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] avg=65536.0000 max=66192 max_rank=2 min=64831 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] avg=65536.0000 max=66192 max_rank=2 min=64831 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[131022, 59898, 98115, 32777, 65553, 65561, 35881, 35481] avg=65536.0000 max=131022 max_rank=0 min=32777 min_rank=3 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] avg=65536.0000 max=66192 max_rank=1 min=64841 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] avg=65536.0000 max=66192 max_rank=1 min=64841 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] avg=65536.0000 max=66192 max_rank=1 min=64841 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[64883, 64897, 66192, 66192, 66192, 64831, 65550, 65551] avg=65536.0000 max=66192 max_rank=2 min=64831 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64840 max_load_reduction_pct=0.494841 moved_to_replicas=257952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] avg=65536.0000 max=66192 max_rank=1 min=64841 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62871 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64830 max_load_reduction_pct=0.494802 moved_to_replicas=193180 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[32784, 98287, 65553, 32802, 65558, 65513, 28, 163763] avg=65536.0000 max=163763 max_rank=7 min=28 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[32784, 98287, 65553, 32802, 65558, 65513, 28, 163763] avg=65536.0000 max=163763 max_rank=7 min=28 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[131022, 59898, 98115, 32777, 65553, 65561, 35881, 35481] avg=65536.0000 max=131022 max_rank=0 min=32777 min_rank=3 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[131022, 59898, 98115, 32777, 65553, 65561, 35881, 35481] avg=65536.0000 max=131022 max_rank=0 min=32777 min_rank=3 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[131022, 59898, 98115, 32777, 65553, 65561, 35881, 35481] avg=65536.0000 max=131022 max_rank=0 min=32777 min_rank=3 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[32784, 98287, 65553, 32802, 65558, 65513, 28, 163763] avg=65536.0000 max=163763 max_rank=7 min=28 min_rank=6 imbalance_ratio=2.498825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[131022, 59898, 98115, 32777, 65553, 65561, 35881, 35481] avg=65536.0000 max=131022 max_rank=0 min=32777 min_rank=3 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] avg=65536.0000 max=66192 max_rank=1 min=64841 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[64997, 252, 98590, 98290, 65552, 32811, 131004, 32792] avg=65536.0000 max=131004 max_rank=6 min=252 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] avg=65536.0000 max=66192 max_rank=1 min=64841 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62871 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64830 max_load_reduction_pct=0.494802 moved_to_replicas=193180 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62871 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64830 max_load_reduction_pct=0.494802 moved_to_replicas=193180 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62871 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64830 max_load_reduction_pct=0.494802 moved_to_replicas=193180 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[65542, 66192, 65553, 64897, 65558, 65513, 64841, 66192] avg=65536.0000 max=66192 max_rank=1 min=64841 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97571 max_load_reduction_pct=0.595806 moved_to_replicas=259240 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62871 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64830 max_load_reduction_pct=0.494802 moved_to_replicas=193180 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] avg=65536.0000 max=66192 max_rank=2 min=64844 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=258907 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[131022, 59898, 98115, 32777, 65553, 65561, 35881, 35481] avg=65536.0000 max=131022 max_rank=0 min=32777 min_rank=3 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[64997, 252, 98590, 98290, 65552, 32811, 131004, 32792] avg=65536.0000 max=131004 max_rank=6 min=252 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[131022, 59898, 98115, 32777, 65553, 65561, 35881, 35481] avg=65536.0000 max=131022 max_rank=0 min=32777 min_rank=3 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[64997, 252, 98590, 98290, 65552, 32811, 131004, 32792] avg=65536.0000 max=131004 max_rank=6 min=252 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[64997, 252, 98590, 98290, 65552, 32811, 131004, 32792] avg=65536.0000 max=131004 max_rank=6 min=252 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[131022, 59898, 98115, 32777, 65553, 65561, 35881, 35481] avg=65536.0000 max=131022 max_rank=0 min=32777 min_rank=3 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[64997, 252, 98590, 98290, 65552, 32811, 131004, 32792] avg=65536.0000 max=131004 max_rank=6 min=252 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 159373, 69907, 32771, 32778, 65540] avg=65536.0000 max=159373 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.431839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62871 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64830 max_load_reduction_pct=0.494802 moved_to_replicas=193180 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] avg=65536.0000 max=66192 max_rank=2 min=64844 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=258907 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] avg=65536.0000 max=66192 max_rank=2 min=64844 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=258907 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62871 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64830 max_load_reduction_pct=0.494802 moved_to_replicas=193180 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] avg=65536.0000 max=66192 max_rank=2 min=64844 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=258907 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] avg=65536.0000 max=66192 max_rank=2 min=64844 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=258907 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[66192, 62871, 66192, 65535, 65553, 65561, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=62871 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64830 max_load_reduction_pct=0.494802 moved_to_replicas=193180 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] avg=65536.0000 max=66192 max_rank=2 min=64156 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=93181 max_load_reduction_pct=0.584672 moved_to_replicas=323351 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[64997, 252, 98590, 98290, 65552, 32811, 131004, 32792] avg=65536.0000 max=131004 max_rank=6 min=252 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 159373, 69907, 32771, 32778, 65540] avg=65536.0000 max=159373 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.431839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 159373, 69907, 32771, 32778, 65540] avg=65536.0000 max=159373 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.431839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[64997, 252, 98590, 98290, 65552, 32811, 131004, 32792] avg=65536.0000 max=131004 max_rank=6 min=252 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 159373, 69907, 32771, 32778, 65540] avg=65536.0000 max=159373 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.431839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 159373, 69907, 32771, 32778, 65540] avg=65536.0000 max=159373 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.431839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[64997, 252, 98590, 98290, 65552, 32811, 131004, 32792] avg=65536.0000 max=131004 max_rank=6 min=252 min_rank=1 imbalance_ratio=1.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] avg=65536.0000 max=66192 max_rank=2 min=64844 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=258907 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] avg=65536.0000 max=66192 max_rank=2 min=64156 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=93181 max_load_reduction_pct=0.584672 moved_to_replicas=323351 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] avg=65536.0000 max=66192 max_rank=2 min=64156 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=93181 max_load_reduction_pct=0.584672 moved_to_replicas=323351 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] avg=65536.0000 max=66192 max_rank=2 min=64156 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=93181 max_load_reduction_pct=0.584672 moved_to_replicas=323351 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] avg=65536.0000 max=66192 max_rank=2 min=64844 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=258907 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] avg=65536.0000 max=66192 max_rank=2 min=64156 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=93181 max_load_reduction_pct=0.584672 moved_to_replicas=323351 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[64997, 65110, 66192, 66192, 65552, 65209, 66192, 64844] avg=65536.0000 max=66192 max_rank=2 min=64844 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64812 max_load_reduction_pct=0.494733 moved_to_replicas=258907 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 159373, 69907, 32771, 32778, 65540] avg=65536.0000 max=159373 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.431839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 159373, 69907, 32771, 32778, 65540] avg=65536.0000 max=159373 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.431839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[64, 32816, 131039, 159373, 69907, 32771, 32778, 65540] avg=65536.0000 max=159373 max_rank=3 min=64 min_rank=0 imbalance_ratio=2.431839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] avg=65536.0000 max=66192 max_rank=2 min=64156 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=93181 max_load_reduction_pct=0.584672 moved_to_replicas=323351 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] avg=65536.0000 max=66192 max_rank=2 min=64156 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=93181 max_load_reduction_pct=0.584672 moved_to_replicas=323351 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[64911, 65576, 66192, 66191, 66192, 65530, 64156, 65540] avg=65536.0000 max=66192 max_rank=2 min=64156 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=93181 max_load_reduction_pct=0.584672 moved_to_replicas=323351 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=2075097 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=525312 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=2102443 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=497024 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=2092887 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508672 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=2097198 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=504320 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=2097766 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=504064 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=2106063 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=496384 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=2104333 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=498176 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=2101429 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=500224 peak_demand=66192 required_capacity=1.010010 + [2026-08-30 05:14:11] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 540.3 | throughput per GPU (TFLOP/s/GPU): 617.0 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.004390E+01 | loss scale: 1.0 | grad norm: 7.314 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=81920 expected_recv=129269 expected_overflow=0 exact_recv_per_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=81920 expected_recv=130135 expected_overflow=0 exact_recv_per_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=81920 expected_recv=129269 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=81920 expected_recv=130135 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=81920 expected_recv=127252 expected_overflow=0 exact_recv_per_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=81920 expected_recv=127252 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=128512 slack_slots=35328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131207 expected_overflow=0 exact_recv_per_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=81920 expected_recv=129946 expected_overflow=0 exact_recv_per_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131207 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=81920 expected_recv=130096 expected_overflow=0 exact_recv_per_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=81920 expected_recv=129946 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131052 expected_overflow=0 exact_recv_per_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=81920 expected_recv=129126 expected_overflow=0 exact_recv_per_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=81920 expected_recv=130096 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131052 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=81920 expected_recv=129126 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=81920 expected_recv=129789 expected_overflow=0 exact_recv_per_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=81920 expected_recv=129789 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131118 expected_overflow=0 exact_recv_per_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=81920 expected_recv=126999 expected_overflow=0 exact_recv_per_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131118 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131133 expected_overflow=0 exact_recv_per_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=81920 expected_recv=126999 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=128512 slack_slots=35328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131133 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131121 expected_overflow=0 exact_recv_per_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131121 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130241 expected_overflow=0 exact_recv_per_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130042 expected_overflow=0 exact_recv_per_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130241 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=81920 expected_recv=129123 expected_overflow=0 exact_recv_per_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130042 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130897 expected_overflow=0 exact_recv_per_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=81920 expected_recv=129123 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130897 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=130048 slack_slots=33792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130637 expected_overflow=0 exact_recv_per_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130637 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131058 expected_overflow=0 exact_recv_per_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131058 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130482 expected_overflow=0 exact_recv_per_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131019 expected_overflow=0 exact_recv_per_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130482 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=81920 expected_recv=128228 expected_overflow=0 exact_recv_per_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131019 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=81920 expected_recv=129758 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=81920 expected_recv=128228 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=81920 expected_recv=129758 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=129024 slack_slots=34816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=81920 expected_recv=129798 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=81920 expected_recv=128349 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=81920 expected_recv=131135 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=81920 expected_recv=129798 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=81920 expected_recv=128349 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=81920 expected_recv=131135 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=81920 expected_recv=129453 expected_overflow=0 exact_recv_per_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=129024 slack_slots=34816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=81920 expected_recv=129453 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=81920 expected_recv=130800 expected_overflow=0 exact_recv_per_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=81920 expected_recv=130800 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=81920 expected_recv=130713 expected_overflow=0 exact_recv_per_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=81920 expected_recv=130713 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=81920 expected_recv=129409 expected_overflow=0 exact_recv_per_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131088 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131049 expected_overflow=0 exact_recv_per_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=81920 expected_recv=129409 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131088 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131049 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131073 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131073 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131150 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130117 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131150 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130117 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=81920 expected_recv=129652 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=81920 expected_recv=129083 expected_overflow=0 exact_recv_per_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130728 expected_overflow=0 exact_recv_per_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=81920 expected_recv=129652 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131087 expected_overflow=0 exact_recv_per_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=81920 expected_recv=129083 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130728 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131087 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=81920 expected_recv=129052 expected_overflow=0 exact_recv_per_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=81920 expected_recv=129052 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131109 expected_overflow=0 exact_recv_per_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131093 expected_overflow=0 exact_recv_per_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131109 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131093 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=81920 expected_recv=129731 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=81920 expected_recv=129943 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=81920 expected_recv=129731 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=81920 expected_recv=129943 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=81920 expected_recv=132388 expected_overflow=0 exact_recv_per_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=81920 expected_recv=129958 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=81920 expected_recv=130752 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131040 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=81920 expected_recv=132388 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=81920 expected_recv=129958 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=81920 expected_recv=130752 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131040 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=81920 expected_recv=127926 expected_overflow=0 exact_recv_per_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=81920 expected_recv=127926 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=81920 expected_recv=128162 expected_overflow=0 exact_recv_per_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=128640 slack_slots=35200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131070 expected_overflow=0 exact_recv_per_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=81920 expected_recv=128162 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131070 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=128640 slack_slots=35200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=81920 expected_recv=128601 expected_overflow=0 exact_recv_per_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=81920 expected_recv=132380 expected_overflow=0 exact_recv_per_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131882 expected_overflow=0 exact_recv_per_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=81920 expected_recv=128601 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=81920 expected_recv=132380 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131882 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=81920 expected_recv=130506 expected_overflow=0 exact_recv_per_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=81920 expected_recv=130946 expected_overflow=0 exact_recv_per_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=81920 expected_recv=130506 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=81920 expected_recv=130946 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131186 expected_overflow=0 exact_recv_per_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=81920 expected_recv=129720 expected_overflow=0 exact_recv_per_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131186 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131651 expected_overflow=0 exact_recv_per_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=81920 expected_recv=129720 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131651 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=81920 expected_recv=127269 expected_overflow=0 exact_recv_per_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=81920 expected_recv=127269 actual_valid_recv=127872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131114 expected_overflow=0 exact_recv_per_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=127872 slack_slots=35968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131114 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131099 expected_overflow=0 exact_recv_per_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131252 expected_overflow=0 exact_recv_per_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131891 expected_overflow=0 exact_recv_per_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131099 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131252 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=81920 expected_recv=129786 expected_overflow=0 exact_recv_per_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131891 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=81920 expected_recv=129786 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=81920 expected_recv=129737 expected_overflow=0 exact_recv_per_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=81920 expected_recv=129737 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=81920 expected_recv=129820 expected_overflow=0 exact_recv_per_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=81920 expected_recv=129717 expected_overflow=0 exact_recv_per_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=81920 expected_recv=129820 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=81920 expected_recv=129799 expected_overflow=0 exact_recv_per_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=81920 expected_recv=129717 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=81920 expected_recv=129799 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131069 expected_overflow=0 exact_recv_per_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131069 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=81920 expected_recv=129700 expected_overflow=0 exact_recv_per_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131071 expected_overflow=0 exact_recv_per_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=81920 expected_recv=129700 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131071 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131030 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131030 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131083 expected_overflow=0 exact_recv_per_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131065 expected_overflow=0 exact_recv_per_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131083 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131065 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[39970, 57568, 53745, 89300, 59772, 77213, 57478, 89242] avg=65536.0000 max=89300 max_rank=3 min=39970 min_rank=0 imbalance_ratio=1.362610 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=81920 expected_recv=129586 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131070 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=81920 expected_recv=130962 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=81920 expected_recv=129586 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131070 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=81920 expected_recv=128776 expected_overflow=0 exact_recv_per_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=81920 expected_recv=130962 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=81920 expected_recv=128776 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=1 min=63078 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23107 max_load_reduction_pct=0.258757 moved_to_replicas=112620 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[39970, 57568, 53745, 89300, 59772, 77213, 57478, 89242] avg=65536.0000 max=89300 max_rank=3 min=39970 min_rank=0 imbalance_ratio=1.362610 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[39970, 57568, 53745, 89300, 59772, 77213, 57478, 89242] avg=65536.0000 max=89300 max_rank=3 min=39970 min_rank=0 imbalance_ratio=1.362610 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=129792 slack_slots=34048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[39970, 57568, 53745, 89300, 59772, 77213, 57478, 89242] avg=65536.0000 max=89300 max_rank=3 min=39970 min_rank=0 imbalance_ratio=1.362610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=1 min=63078 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23107 max_load_reduction_pct=0.258757 moved_to_replicas=112620 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[39970, 57568, 53745, 89300, 59772, 77213, 57478, 89242] avg=65536.0000 max=89300 max_rank=3 min=39970 min_rank=0 imbalance_ratio=1.362610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32888, 38624, 38, 163393, 44313, 32826, 65517, 146689] avg=65536.0000 max=163393 max_rank=3 min=38 min_rank=2 imbalance_ratio=2.493179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=1 min=63078 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23107 max_load_reduction_pct=0.258757 moved_to_replicas=112620 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[39970, 57568, 53745, 89300, 59772, 77213, 57478, 89242] avg=65536.0000 max=89300 max_rank=3 min=39970 min_rank=0 imbalance_ratio=1.362610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=1 min=63078 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23107 max_load_reduction_pct=0.258757 moved_to_replicas=112620 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[39970, 57568, 53745, 89300, 59772, 77213, 57478, 89242] avg=65536.0000 max=89300 max_rank=3 min=39970 min_rank=0 imbalance_ratio=1.362610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[39970, 57568, 53745, 89300, 59772, 77213, 57478, 89242] avg=65536.0000 max=89300 max_rank=3 min=39970 min_rank=0 imbalance_ratio=1.362610 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=1 min=63078 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23107 max_load_reduction_pct=0.258757 moved_to_replicas=112620 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] avg=65536.0000 max=66192 max_rank=3 min=64492 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.594891 moved_to_replicas=351628 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32888, 38624, 38, 163393, 44313, 32826, 65517, 146689] avg=65536.0000 max=163393 max_rank=3 min=38 min_rank=2 imbalance_ratio=2.493179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=1 min=63078 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23107 max_load_reduction_pct=0.258757 moved_to_replicas=112620 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32888, 38624, 38, 163393, 44313, 32826, 65517, 146689] avg=65536.0000 max=163393 max_rank=3 min=38 min_rank=2 imbalance_ratio=2.493179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=1 min=63078 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23107 max_load_reduction_pct=0.258757 moved_to_replicas=112620 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32888, 38624, 38, 163393, 44313, 32826, 65517, 146689] avg=65536.0000 max=163393 max_rank=3 min=38 min_rank=2 imbalance_ratio=2.493179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[63078, 66193, 66191, 66192, 64057, 66192, 66192, 66193] avg=65536.0000 max=66193 max_rank=1 min=63078 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=23107 max_load_reduction_pct=0.258757 moved_to_replicas=112620 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] avg=65536.0000 max=66192 max_rank=3 min=64492 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.594891 moved_to_replicas=351628 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32888, 38624, 38, 163393, 44313, 32826, 65517, 146689] avg=65536.0000 max=163393 max_rank=3 min=38 min_rank=2 imbalance_ratio=2.493179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32782, 65570, 41349, 97881, 66649, 98306, 65585, 56166] avg=65536.0000 max=98306 max_rank=5 min=32782 min_rank=0 imbalance_ratio=1.500031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] avg=65536.0000 max=66192 max_rank=3 min=64492 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.594891 moved_to_replicas=351628 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32888, 38624, 38, 163393, 44313, 32826, 65517, 146689] avg=65536.0000 max=163393 max_rank=3 min=38 min_rank=2 imbalance_ratio=2.493179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] avg=65536.0000 max=66192 max_rank=3 min=64492 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.594891 moved_to_replicas=351628 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32888, 38624, 38, 163393, 44313, 32826, 65517, 146689] avg=65536.0000 max=163393 max_rank=3 min=38 min_rank=2 imbalance_ratio=2.493179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32888, 38624, 38, 163393, 44313, 32826, 65517, 146689] avg=65536.0000 max=163393 max_rank=3 min=38 min_rank=2 imbalance_ratio=2.493179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] avg=65536.0000 max=66192 max_rank=3 min=64492 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.594891 moved_to_replicas=351628 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] avg=65536.0000 max=66193 max_rank=2 min=63468 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32113 max_load_reduction_pct=0.326664 moved_to_replicas=128580 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[32782, 65570, 41349, 97881, 66649, 98306, 65585, 56166] avg=65536.0000 max=98306 max_rank=5 min=32782 min_rank=0 imbalance_ratio=1.500031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] avg=65536.0000 max=66192 max_rank=3 min=64492 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.594891 moved_to_replicas=351628 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[32782, 65570, 41349, 97881, 66649, 98306, 65585, 56166] avg=65536.0000 max=98306 max_rank=5 min=32782 min_rank=0 imbalance_ratio=1.500031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] avg=65536.0000 max=66192 max_rank=3 min=64492 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.594891 moved_to_replicas=351628 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32782, 65570, 41349, 97881, 66649, 98306, 65585, 56166] avg=65536.0000 max=98306 max_rank=5 min=32782 min_rank=0 imbalance_ratio=1.500031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[65639, 66191, 65527, 66192, 64492, 64538, 65517, 66192] avg=65536.0000 max=66192 max_rank=3 min=64492 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=97201 max_load_reduction_pct=0.594891 moved_to_replicas=351628 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] avg=65536.0000 max=66193 max_rank=2 min=63468 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32113 max_load_reduction_pct=0.326664 moved_to_replicas=128580 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[32782, 65570, 41349, 97881, 66649, 98306, 65585, 56166] avg=65536.0000 max=98306 max_rank=5 min=32782 min_rank=0 imbalance_ratio=1.500031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[64, 33045, 32832, 65425, 98280, 98251, 163492, 32899] avg=65536.0000 max=163492 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.494690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] avg=65536.0000 max=66193 max_rank=2 min=63468 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32113 max_load_reduction_pct=0.326664 moved_to_replicas=128580 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32782, 65570, 41349, 97881, 66649, 98306, 65585, 56166] avg=65536.0000 max=98306 max_rank=5 min=32782 min_rank=0 imbalance_ratio=1.500031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] avg=65536.0000 max=66193 max_rank=2 min=63468 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32113 max_load_reduction_pct=0.326664 moved_to_replicas=128580 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32782, 65570, 41349, 97881, 66649, 98306, 65585, 56166] avg=65536.0000 max=98306 max_rank=5 min=32782 min_rank=0 imbalance_ratio=1.500031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32782, 65570, 41349, 97881, 66649, 98306, 65585, 56166] avg=65536.0000 max=98306 max_rank=5 min=32782 min_rank=0 imbalance_ratio=1.500031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] avg=65536.0000 max=66193 max_rank=2 min=63468 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32113 max_load_reduction_pct=0.326664 moved_to_replicas=128580 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[64, 33045, 32832, 65425, 98280, 98251, 163492, 32899] avg=65536.0000 max=163492 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.494690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] avg=65536.0000 max=66192 max_rank=4 min=64612 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97300 max_load_reduction_pct=0.595136 moved_to_replicas=322429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] avg=65536.0000 max=66193 max_rank=2 min=63468 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32113 max_load_reduction_pct=0.326664 moved_to_replicas=128580 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] avg=65536.0000 max=66193 max_rank=2 min=63468 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32113 max_load_reduction_pct=0.326664 moved_to_replicas=128580 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[64, 33045, 32832, 65425, 98280, 98251, 163492, 32899] avg=65536.0000 max=163492 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.494690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[64896, 65570, 66193, 66192, 66192, 66192, 65585, 63468] avg=65536.0000 max=66193 max_rank=2 min=63468 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=32113 max_load_reduction_pct=0.326664 moved_to_replicas=128580 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[64, 33045, 32832, 65425, 98280, 98251, 163492, 32899] avg=65536.0000 max=163492 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.494690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] avg=65536.0000 max=66192 max_rank=4 min=64612 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97300 max_load_reduction_pct=0.595136 moved_to_replicas=322429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[64, 33045, 32832, 65425, 98280, 98251, 163492, 32899] avg=65536.0000 max=163492 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.494690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32785, 65528, 32296, 73203, 99712, 98264, 57004, 65496] avg=65536.0000 max=99712 max_rank=4 min=32296 min_rank=2 imbalance_ratio=1.521484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] avg=65536.0000 max=66192 max_rank=4 min=64612 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97300 max_load_reduction_pct=0.595136 moved_to_replicas=322429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[64, 33045, 32832, 65425, 98280, 98251, 163492, 32899] avg=65536.0000 max=163492 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.494690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] avg=65536.0000 max=66192 max_rank=4 min=64612 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97300 max_load_reduction_pct=0.595136 moved_to_replicas=322429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[64, 33045, 32832, 65425, 98280, 98251, 163492, 32899] avg=65536.0000 max=163492 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.494690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] avg=65536.0000 max=66192 max_rank=4 min=64612 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97300 max_load_reduction_pct=0.595136 moved_to_replicas=322429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[64, 33045, 32832, 65425, 98280, 98251, 163492, 32899] avg=65536.0000 max=163492 max_rank=6 min=64 min_rank=0 imbalance_ratio=2.494690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32785, 65528, 32296, 73203, 99712, 98264, 57004, 65496] avg=65536.0000 max=99712 max_rank=4 min=32296 min_rank=2 imbalance_ratio=1.521484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] avg=65536.0000 max=66192 max_rank=3 min=64015 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33520 max_load_reduction_pct=0.336168 moved_to_replicas=141324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] avg=65536.0000 max=66192 max_rank=4 min=64612 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97300 max_load_reduction_pct=0.595136 moved_to_replicas=322429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] avg=65536.0000 max=66192 max_rank=4 min=64612 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97300 max_load_reduction_pct=0.595136 moved_to_replicas=322429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32785, 65528, 32296, 73203, 99712, 98264, 57004, 65496] avg=65536.0000 max=99712 max_rank=4 min=32296 min_rank=2 imbalance_ratio=1.521484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32785, 65528, 32296, 73203, 99712, 98264, 57004, 65496] avg=65536.0000 max=99712 max_rank=4 min=32296 min_rank=2 imbalance_ratio=1.521484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[65584, 65104, 64612, 65425, 66192, 66192, 66192, 64987] avg=65536.0000 max=66192 max_rank=4 min=64612 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=97300 max_load_reduction_pct=0.595136 moved_to_replicas=322429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] avg=65536.0000 max=66192 max_rank=3 min=64015 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33520 max_load_reduction_pct=0.336168 moved_to_replicas=141324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32785, 65528, 32296, 73203, 99712, 98264, 57004, 65496] avg=65536.0000 max=99712 max_rank=4 min=32296 min_rank=2 imbalance_ratio=1.521484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34447, 98143, 65583, 98291, 98287, 63962, 32802] avg=65536.0000 max=98291 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] avg=65536.0000 max=66192 max_rank=3 min=64015 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33520 max_load_reduction_pct=0.336168 moved_to_replicas=141324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] avg=65536.0000 max=66192 max_rank=3 min=64015 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33520 max_load_reduction_pct=0.336168 moved_to_replicas=141324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32785, 65528, 32296, 73203, 99712, 98264, 57004, 65496] avg=65536.0000 max=99712 max_rank=4 min=32296 min_rank=2 imbalance_ratio=1.521484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32785, 65528, 32296, 73203, 99712, 98264, 57004, 65496] avg=65536.0000 max=99712 max_rank=4 min=32296 min_rank=2 imbalance_ratio=1.521484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] avg=65536.0000 max=66192 max_rank=3 min=64015 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33520 max_load_reduction_pct=0.336168 moved_to_replicas=141324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34447, 98143, 65583, 98291, 98287, 63962, 32802] avg=65536.0000 max=98291 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32785, 65528, 32296, 73203, 99712, 98264, 57004, 65496] avg=65536.0000 max=99712 max_rank=4 min=32296 min_rank=2 imbalance_ratio=1.521484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] avg=65536.0000 max=66192 max_rank=1 min=64168 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32099 max_load_reduction_pct=0.326571 moved_to_replicas=192370 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] avg=65536.0000 max=66192 max_rank=3 min=64015 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33520 max_load_reduction_pct=0.336168 moved_to_replicas=141324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] avg=65536.0000 max=66192 max_rank=3 min=64015 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33520 max_load_reduction_pct=0.336168 moved_to_replicas=141324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34447, 98143, 65583, 98291, 98287, 63962, 32802] avg=65536.0000 max=98291 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34447, 98143, 65583, 98291, 98287, 63962, 32802] avg=65536.0000 max=98291 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] avg=65536.0000 max=66192 max_rank=1 min=64168 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32099 max_load_reduction_pct=0.326571 moved_to_replicas=192370 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[65617, 65528, 65056, 66192, 66192, 66192, 64015, 65496] avg=65536.0000 max=66192 max_rank=3 min=64015 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=33520 max_load_reduction_pct=0.336168 moved_to_replicas=141324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34447, 98143, 65583, 98291, 98287, 63962, 32802] avg=65536.0000 max=98291 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32796, 65392, 32668, 32771, 65520, 67981, 131336, 95824] avg=65536.0000 max=131336 max_rank=6 min=32668 min_rank=2 imbalance_ratio=2.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] avg=65536.0000 max=66192 max_rank=1 min=64168 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32099 max_load_reduction_pct=0.326571 moved_to_replicas=192370 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34447, 98143, 65583, 98291, 98287, 63962, 32802] avg=65536.0000 max=98291 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34447, 98143, 65583, 98291, 98287, 63962, 32802] avg=65536.0000 max=98291 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] avg=65536.0000 max=66192 max_rank=1 min=64168 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32099 max_load_reduction_pct=0.326571 moved_to_replicas=192370 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] avg=65536.0000 max=66192 max_rank=1 min=64168 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32099 max_load_reduction_pct=0.326571 moved_to_replicas=192370 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32796, 65392, 32668, 32771, 65520, 67981, 131336, 95824] avg=65536.0000 max=131336 max_rank=6 min=32668 min_rank=2 imbalance_ratio=2.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32773, 34447, 98143, 65583, 98291, 98287, 63962, 32802] avg=65536.0000 max=98291 max_rank=4 min=32773 min_rank=0 imbalance_ratio=1.499802 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64217 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65144 max_load_reduction_pct=0.496010 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] avg=65536.0000 max=66192 max_rank=1 min=64168 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32099 max_load_reduction_pct=0.326571 moved_to_replicas=192370 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] avg=65536.0000 max=66192 max_rank=1 min=64168 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32099 max_load_reduction_pct=0.326571 moved_to_replicas=192370 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32796, 65392, 32668, 32771, 65520, 67981, 131336, 95824] avg=65536.0000 max=131336 max_rank=6 min=32668 min_rank=2 imbalance_ratio=2.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32796, 65392, 32668, 32771, 65520, 67981, 131336, 95824] avg=65536.0000 max=131336 max_rank=6 min=32668 min_rank=2 imbalance_ratio=2.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64217 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65144 max_load_reduction_pct=0.496010 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[64872, 66192, 66192, 65583, 66192, 66192, 64168, 64897] avg=65536.0000 max=66192 max_rank=1 min=64168 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32099 max_load_reduction_pct=0.326571 moved_to_replicas=192370 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32796, 65392, 32668, 32771, 65520, 67981, 131336, 95824] avg=65536.0000 max=131336 max_rank=6 min=32668 min_rank=2 imbalance_ratio=2.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 130982, 98466, 65329, 21, 32811, 65593] avg=65536.0000 max=130982 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32796, 65392, 32668, 32771, 65520, 67981, 131336, 95824] avg=65536.0000 max=131336 max_rank=6 min=32668 min_rank=2 imbalance_ratio=2.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64217 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65144 max_load_reduction_pct=0.496010 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32796, 65392, 32668, 32771, 65520, 67981, 131336, 95824] avg=65536.0000 max=131336 max_rank=6 min=32668 min_rank=2 imbalance_ratio=2.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64217 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65144 max_load_reduction_pct=0.496010 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64217 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65144 max_load_reduction_pct=0.496010 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] avg=65536.0000 max=66192 max_rank=2 min=64811 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64790 max_load_reduction_pct=0.494648 moved_to_replicas=194138 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 130982, 98466, 65329, 21, 32811, 65593] avg=65536.0000 max=130982 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32796, 65392, 32668, 32771, 65520, 67981, 131336, 95824] avg=65536.0000 max=131336 max_rank=6 min=32668 min_rank=2 imbalance_ratio=2.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64217 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65144 max_load_reduction_pct=0.496010 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64217 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65144 max_load_reduction_pct=0.496010 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 130982, 98466, 65329, 21, 32811, 65593] avg=65536.0000 max=130982 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 130982, 98466, 65329, 21, 32811, 65593] avg=65536.0000 max=130982 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[64217, 65392, 65424, 65159, 65520, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=5 min=64217 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65144 max_load_reduction_pct=0.496010 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] avg=65536.0000 max=66192 max_rank=2 min=64811 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64790 max_load_reduction_pct=0.494648 moved_to_replicas=194138 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 130982, 98466, 65329, 21, 32811, 65593] avg=65536.0000 max=130982 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[44, 65536, 55, 130953, 65552, 65559, 98290, 98299] avg=65536.0000 max=130953 max_rank=3 min=44 min_rank=0 imbalance_ratio=1.998184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 130982, 98466, 65329, 21, 32811, 65593] avg=65536.0000 max=130982 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 130982, 98466, 65329, 21, 32811, 65593] avg=65536.0000 max=130982 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] avg=65536.0000 max=66192 max_rank=2 min=64811 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64790 max_load_reduction_pct=0.494648 moved_to_replicas=194138 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] avg=65536.0000 max=66192 max_rank=2 min=64811 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64790 max_load_reduction_pct=0.494648 moved_to_replicas=194138 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] avg=65536.0000 max=66192 max_rank=2 min=64811 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64790 max_load_reduction_pct=0.494648 moved_to_replicas=194138 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64167 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64761 max_load_reduction_pct=0.494536 moved_to_replicas=258002 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 130982, 98466, 65329, 21, 32811, 65593] avg=65536.0000 max=130982 max_rank=2 min=21 min_rank=5 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[44, 65536, 55, 130953, 65552, 65559, 98290, 98299] avg=65536.0000 max=130953 max_rank=3 min=44 min_rank=0 imbalance_ratio=1.998184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] avg=65536.0000 max=66192 max_rank=2 min=64811 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64790 max_load_reduction_pct=0.494648 moved_to_replicas=194138 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] avg=65536.0000 max=66192 max_rank=2 min=64811 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64790 max_load_reduction_pct=0.494648 moved_to_replicas=194138 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[44, 65536, 55, 130953, 65552, 65559, 98290, 98299] avg=65536.0000 max=130953 max_rank=3 min=44 min_rank=0 imbalance_ratio=1.998184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[44, 65536, 55, 130953, 65552, 65559, 98290, 98299] avg=65536.0000 max=130953 max_rank=3 min=44 min_rank=0 imbalance_ratio=1.998184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[65545, 65541, 66192, 66192, 65329, 64811, 65085, 65593] avg=65536.0000 max=66192 max_rank=2 min=64811 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64790 max_load_reduction_pct=0.494648 moved_to_replicas=194138 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64167 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64761 max_load_reduction_pct=0.494536 moved_to_replicas=258002 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[44, 65536, 55, 130953, 65552, 65559, 98290, 98299] avg=65536.0000 max=130953 max_rank=3 min=44 min_rank=0 imbalance_ratio=1.998184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[131019, 98276, 32803, 110, 65393, 65514, 32888, 98285] avg=65536.0000 max=131019 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[44, 65536, 55, 130953, 65552, 65559, 98290, 98299] avg=65536.0000 max=130953 max_rank=3 min=44 min_rank=0 imbalance_ratio=1.998184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[44, 65536, 55, 130953, 65552, 65559, 98290, 98299] avg=65536.0000 max=130953 max_rank=3 min=44 min_rank=0 imbalance_ratio=1.998184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64167 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64761 max_load_reduction_pct=0.494536 moved_to_replicas=258002 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64167 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64761 max_load_reduction_pct=0.494536 moved_to_replicas=258002 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64167 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64761 max_load_reduction_pct=0.494536 moved_to_replicas=258002 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] avg=65536.0000 max=66192 max_rank=0 min=64874 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.494791 moved_to_replicas=258186 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[44, 65536, 55, 130953, 65552, 65559, 98290, 98299] avg=65536.0000 max=130953 max_rank=3 min=44 min_rank=0 imbalance_ratio=1.998184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[131019, 98276, 32803, 110, 65393, 65514, 32888, 98285] avg=65536.0000 max=131019 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64167 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64761 max_load_reduction_pct=0.494536 moved_to_replicas=258002 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64167 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64761 max_load_reduction_pct=0.494536 moved_to_replicas=258002 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[131019, 98276, 32803, 110, 65393, 65514, 32888, 98285] avg=65536.0000 max=131019 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[131019, 98276, 32803, 110, 65393, 65514, 32888, 98285] avg=65536.0000 max=131019 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[131019, 98276, 32803, 110, 65393, 65514, 32888, 98285] avg=65536.0000 max=131019 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[64898, 65536, 64167, 66192, 65552, 65559, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64167 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64761 max_load_reduction_pct=0.494536 moved_to_replicas=258002 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] avg=65536.0000 max=66192 max_rank=0 min=64874 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.494791 moved_to_replicas=258186 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[109260, 32783, 32806, 33049, 119323, 66076, 98210, 32781] avg=65536.0000 max=119323 max_rank=4 min=32781 min_rank=7 imbalance_ratio=1.820724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[131019, 98276, 32803, 110, 65393, 65514, 32888, 98285] avg=65536.0000 max=131019 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[131019, 98276, 32803, 110, 65393, 65514, 32888, 98285] avg=65536.0000 max=131019 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] avg=65536.0000 max=66192 max_rank=0 min=64874 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.494791 moved_to_replicas=258186 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] avg=65536.0000 max=66192 max_rank=0 min=64874 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.494791 moved_to_replicas=258186 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] avg=65536.0000 max=66192 max_rank=0 min=64874 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.494791 moved_to_replicas=258186 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] avg=65536.0000 max=66194 max_rank=4 min=62398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=53129 max_load_reduction_pct=0.445254 moved_to_replicas=256908 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] avg=65536.0000 max=66192 max_rank=0 min=64874 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.494791 moved_to_replicas=258186 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] avg=65536.0000 max=66192 max_rank=0 min=64874 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.494791 moved_to_replicas=258186 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[131019, 98276, 32803, 110, 65393, 65514, 32888, 98285] avg=65536.0000 max=131019 max_rank=0 min=110 min_rank=3 imbalance_ratio=1.999191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[109260, 32783, 32806, 33049, 119323, 66076, 98210, 32781] avg=65536.0000 max=119323 max_rank=4 min=32781 min_rank=7 imbalance_ratio=1.820724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[109260, 32783, 32806, 33049, 119323, 66076, 98210, 32781] avg=65536.0000 max=119323 max_rank=4 min=32781 min_rank=7 imbalance_ratio=1.820724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[109260, 32783, 32806, 33049, 119323, 66076, 98210, 32781] avg=65536.0000 max=119323 max_rank=4 min=32781 min_rank=7 imbalance_ratio=1.820724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[109260, 32783, 32806, 33049, 119323, 66076, 98210, 32781] avg=65536.0000 max=119323 max_rank=4 min=32781 min_rank=7 imbalance_ratio=1.820724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[66192, 66192, 64874, 64959, 65393, 65514, 64972, 66192] avg=65536.0000 max=66192 max_rank=0 min=64874 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.494791 moved_to_replicas=258186 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[82, 98245, 64949, 98317, 32815, 66111, 98304, 65465] avg=65536.0000 max=98317 max_rank=3 min=82 min_rank=0 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] avg=65536.0000 max=66194 max_rank=4 min=62398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=53129 max_load_reduction_pct=0.445254 moved_to_replicas=256908 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[109260, 32783, 32806, 33049, 119323, 66076, 98210, 32781] avg=65536.0000 max=119323 max_rank=4 min=32781 min_rank=7 imbalance_ratio=1.820724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[109260, 32783, 32806, 33049, 119323, 66076, 98210, 32781] avg=65536.0000 max=119323 max_rank=4 min=32781 min_rank=7 imbalance_ratio=1.820724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] avg=65536.0000 max=66194 max_rank=4 min=62398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=53129 max_load_reduction_pct=0.445254 moved_to_replicas=256908 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] avg=65536.0000 max=66194 max_rank=4 min=62398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=53129 max_load_reduction_pct=0.445254 moved_to_replicas=256908 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] avg=65536.0000 max=66194 max_rank=4 min=62398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=53129 max_load_reduction_pct=0.445254 moved_to_replicas=256908 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] avg=65536.0000 max=66192 max_rank=1 min=64319 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=192617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] avg=65536.0000 max=66194 max_rank=4 min=62398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=53129 max_load_reduction_pct=0.445254 moved_to_replicas=256908 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] avg=65536.0000 max=66194 max_rank=4 min=62398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=53129 max_load_reduction_pct=0.445254 moved_to_replicas=256908 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[109260, 32783, 32806, 33049, 119323, 66076, 98210, 32781] avg=65536.0000 max=119323 max_rank=4 min=32781 min_rank=7 imbalance_ratio=1.820724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[82, 98245, 64949, 98317, 32815, 66111, 98304, 65465] avg=65536.0000 max=98317 max_rank=3 min=82 min_rank=0 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[82, 98245, 64949, 98317, 32815, 66111, 98304, 65465] avg=65536.0000 max=98317 max_rank=3 min=82 min_rank=0 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[82, 98245, 64949, 98317, 32815, 66111, 98304, 65465] avg=65536.0000 max=98317 max_rank=3 min=82 min_rank=0 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32848, 48861, 65578, 65678, 32809, 33800, 81124, 163590] avg=65536.0000 max=163590 max_rank=7 min=32809 min_rank=4 imbalance_ratio=2.496185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[82, 98245, 64949, 98317, 32815, 66111, 98304, 65465] avg=65536.0000 max=98317 max_rank=3 min=82 min_rank=0 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[66193, 62398, 65509, 66191, 66194, 66076, 66192, 65535] avg=65536.0000 max=66194 max_rank=4 min=62398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=53129 max_load_reduction_pct=0.445254 moved_to_replicas=256908 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] avg=65536.0000 max=66192 max_rank=1 min=64319 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=192617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[82, 98245, 64949, 98317, 32815, 66111, 98304, 65465] avg=65536.0000 max=98317 max_rank=3 min=82 min_rank=0 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[82, 98245, 64949, 98317, 32815, 66111, 98304, 65465] avg=65536.0000 max=98317 max_rank=3 min=82 min_rank=0 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] avg=65536.0000 max=66192 max_rank=1 min=64319 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=192617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] avg=65536.0000 max=66192 max_rank=1 min=64319 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=192617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] avg=65536.0000 max=66192 max_rank=7 min=63795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97398 max_load_reduction_pct=0.595379 moved_to_replicas=221739 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] avg=65536.0000 max=66192 max_rank=1 min=64319 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=192617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] avg=65536.0000 max=66192 max_rank=1 min=64319 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=192617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] avg=65536.0000 max=66192 max_rank=1 min=64319 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=192617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32848, 48861, 65578, 65678, 32809, 33800, 81124, 163590] avg=65536.0000 max=163590 max_rank=7 min=32809 min_rank=4 imbalance_ratio=2.496185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[82, 98245, 64949, 98317, 32815, 66111, 98304, 65465] avg=65536.0000 max=98317 max_rank=3 min=82 min_rank=0 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32848, 48861, 65578, 65678, 32809, 33800, 81124, 163590] avg=65536.0000 max=163590 max_rank=7 min=32809 min_rank=4 imbalance_ratio=2.496185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32848, 48861, 65578, 65678, 32809, 33800, 81124, 163590] avg=65536.0000 max=163590 max_rank=7 min=32809 min_rank=4 imbalance_ratio=2.496185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[98177, 4334, 131116, 32804, 32837, 94009, 98274, 32737] avg=65536.0000 max=131116 max_rank=2 min=4334 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32848, 48861, 65578, 65678, 32809, 33800, 81124, 163590] avg=65536.0000 max=163590 max_rank=7 min=32809 min_rank=4 imbalance_ratio=2.496185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[64319, 66192, 64949, 66192, 64868, 66111, 66192, 65465] avg=65536.0000 max=66192 max_rank=1 min=64319 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=192617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] avg=65536.0000 max=66192 max_rank=7 min=63795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97398 max_load_reduction_pct=0.595379 moved_to_replicas=221739 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32848, 48861, 65578, 65678, 32809, 33800, 81124, 163590] avg=65536.0000 max=163590 max_rank=7 min=32809 min_rank=4 imbalance_ratio=2.496185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32848, 48861, 65578, 65678, 32809, 33800, 81124, 163590] avg=65536.0000 max=163590 max_rank=7 min=32809 min_rank=4 imbalance_ratio=2.496185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] avg=65536.0000 max=66192 max_rank=7 min=63795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97398 max_load_reduction_pct=0.595379 moved_to_replicas=221739 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] avg=65536.0000 max=66192 max_rank=7 min=63795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97398 max_load_reduction_pct=0.595379 moved_to_replicas=221739 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] avg=65536.0000 max=66193 max_rank=5 min=64822 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=317465 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] avg=65536.0000 max=66192 max_rank=7 min=63795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97398 max_load_reduction_pct=0.595379 moved_to_replicas=221739 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] avg=65536.0000 max=66192 max_rank=7 min=63795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97398 max_load_reduction_pct=0.595379 moved_to_replicas=221739 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] avg=65536.0000 max=66192 max_rank=7 min=63795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97398 max_load_reduction_pct=0.595379 moved_to_replicas=221739 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32848, 48861, 65578, 65678, 32809, 33800, 81124, 163590] avg=65536.0000 max=163590 max_rank=7 min=32809 min_rank=4 imbalance_ratio=2.496185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[98177, 4334, 131116, 32804, 32837, 94009, 98274, 32737] avg=65536.0000 max=131116 max_rank=2 min=4334 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[98177, 4334, 131116, 32804, 32837, 94009, 98274, 32737] avg=65536.0000 max=131116 max_rank=2 min=4334 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[98177, 4334, 131116, 32804, 32837, 94009, 98274, 32737] avg=65536.0000 max=131116 max_rank=2 min=4334 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32794, 65541, 32979, 65544, 131079, 32775, 98041, 65535] avg=65536.0000 max=131079 max_rank=4 min=32775 min_rank=5 imbalance_ratio=2.000107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[98177, 4334, 131116, 32804, 32837, 94009, 98274, 32737] avg=65536.0000 max=131116 max_rank=2 min=4334 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] avg=65536.0000 max=66193 max_rank=5 min=64822 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=317465 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[65593, 63795, 65578, 65678, 65563, 65699, 66190, 66192] avg=65536.0000 max=66192 max_rank=7 min=63795 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97398 max_load_reduction_pct=0.595379 moved_to_replicas=221739 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[98177, 4334, 131116, 32804, 32837, 94009, 98274, 32737] avg=65536.0000 max=131116 max_rank=2 min=4334 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[98177, 4334, 131116, 32804, 32837, 94009, 98274, 32737] avg=65536.0000 max=131116 max_rank=2 min=4334 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] avg=65536.0000 max=66193 max_rank=5 min=64822 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=317465 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] avg=65536.0000 max=66193 max_rank=5 min=64822 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=317465 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] avg=65536.0000 max=66192 max_rank=4 min=64828 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64887 max_load_reduction_pct=0.495022 moved_to_replicas=193590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] avg=65536.0000 max=66193 max_rank=5 min=64822 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=317465 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] avg=65536.0000 max=66193 max_rank=5 min=64822 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=317465 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] avg=65536.0000 max=66193 max_rank=5 min=64822 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=317465 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32794, 65541, 32979, 65544, 131079, 32775, 98041, 65535] avg=65536.0000 max=131079 max_rank=4 min=32775 min_rank=5 imbalance_ratio=2.000107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[98177, 4334, 131116, 32804, 32837, 94009, 98274, 32737] avg=65536.0000 max=131116 max_rank=2 min=4334 min_rank=1 imbalance_ratio=2.000671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32794, 65541, 32979, 65544, 131079, 32775, 98041, 65535] avg=65536.0000 max=131079 max_rank=4 min=32775 min_rank=5 imbalance_ratio=2.000107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32794, 65541, 32979, 65544, 131079, 32775, 98041, 65535] avg=65536.0000 max=131079 max_rank=4 min=32775 min_rank=5 imbalance_ratio=2.000107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32794, 65541, 32979, 65544, 131079, 32775, 98041, 65535] avg=65536.0000 max=131079 max_rank=4 min=32775 min_rank=5 imbalance_ratio=2.000107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[65534, 98259, 98492, 65467, 65529, 122, 32833, 98052] avg=65536.0000 max=98492 max_rank=2 min=122 min_rank=5 imbalance_ratio=1.502869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] avg=65536.0000 max=66192 max_rank=4 min=64828 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64887 max_load_reduction_pct=0.495022 moved_to_replicas=193590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32794, 65541, 32979, 65544, 131079, 32775, 98041, 65535] avg=65536.0000 max=131079 max_rank=4 min=32775 min_rank=5 imbalance_ratio=2.000107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[66192, 64905, 66192, 64886, 64822, 66193, 66192, 64906] avg=65536.0000 max=66193 max_rank=5 min=64822 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=64923 max_load_reduction_pct=0.495157 moved_to_replicas=317465 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32794, 65541, 32979, 65544, 131079, 32775, 98041, 65535] avg=65536.0000 max=131079 max_rank=4 min=32775 min_rank=5 imbalance_ratio=2.000107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] avg=65536.0000 max=66192 max_rank=4 min=64828 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64887 max_load_reduction_pct=0.495022 moved_to_replicas=193590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] avg=65536.0000 max=66192 max_rank=4 min=64828 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64887 max_load_reduction_pct=0.495022 moved_to_replicas=193590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] avg=65536.0000 max=66192 max_rank=4 min=64828 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64887 max_load_reduction_pct=0.495022 moved_to_replicas=193590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] avg=65536.0000 max=66192 max_rank=1 min=64489 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32300 max_load_reduction_pct=0.327945 moved_to_replicas=192545 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] avg=65536.0000 max=66192 max_rank=4 min=64828 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64887 max_load_reduction_pct=0.495022 moved_to_replicas=193590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] avg=65536.0000 max=66192 max_rank=4 min=64828 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64887 max_load_reduction_pct=0.495022 moved_to_replicas=193590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[65534, 98259, 98492, 65467, 65529, 122, 32833, 98052] avg=65536.0000 max=98492 max_rank=2 min=122 min_rank=5 imbalance_ratio=1.502869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32794, 65541, 32979, 65544, 131079, 32775, 98041, 65535] avg=65536.0000 max=131079 max_rank=4 min=32775 min_rank=5 imbalance_ratio=2.000107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[65534, 98259, 98492, 65467, 65529, 122, 32833, 98052] avg=65536.0000 max=98492 max_rank=2 min=122 min_rank=5 imbalance_ratio=1.502869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[65534, 98259, 98492, 65467, 65529, 122, 32833, 98052] avg=65536.0000 max=98492 max_rank=2 min=122 min_rank=5 imbalance_ratio=1.502869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[65534, 98259, 98492, 65467, 65529, 122, 32833, 98052] avg=65536.0000 max=98492 max_rank=2 min=122 min_rank=5 imbalance_ratio=1.502869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] avg=65536.0000 max=66192 max_rank=1 min=64489 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32300 max_load_reduction_pct=0.327945 moved_to_replicas=192545 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[65534, 98259, 98492, 65467, 65529, 122, 32833, 98052] avg=65536.0000 max=98492 max_rank=2 min=122 min_rank=5 imbalance_ratio=1.502869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[64924, 65541, 64828, 65544, 66192, 65532, 66192, 65535] avg=65536.0000 max=66192 max_rank=4 min=64828 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64887 max_load_reduction_pct=0.495022 moved_to_replicas=193590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] avg=65536.0000 max=66192 max_rank=1 min=64489 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32300 max_load_reduction_pct=0.327945 moved_to_replicas=192545 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[65534, 98259, 98492, 65467, 65529, 122, 32833, 98052] avg=65536.0000 max=98492 max_rank=2 min=122 min_rank=5 imbalance_ratio=1.502869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] avg=65536.0000 max=66192 max_rank=1 min=64489 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32300 max_load_reduction_pct=0.327945 moved_to_replicas=192545 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] avg=65536.0000 max=66192 max_rank=1 min=64489 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32300 max_load_reduction_pct=0.327945 moved_to_replicas=192545 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] avg=65536.0000 max=66192 max_rank=1 min=64489 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32300 max_load_reduction_pct=0.327945 moved_to_replicas=192545 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] avg=65536.0000 max=66192 max_rank=1 min=64489 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32300 max_load_reduction_pct=0.327945 moved_to_replicas=192545 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[65534, 98259, 98492, 65467, 65529, 122, 32833, 98052] avg=65536.0000 max=98492 max_rank=2 min=122 min_rank=5 imbalance_ratio=1.502869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[65534, 66192, 66192, 65467, 65529, 64489, 64693, 66192] avg=65536.0000 max=66192 max_rank=1 min=64489 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32300 max_load_reduction_pct=0.327945 moved_to_replicas=192545 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=2089198 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=517760 peak_demand=66195 required_capacity=1.010056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=2093480 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=515200 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=2095910 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=510720 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=2099861 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507008 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=2097601 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509312 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=2093751 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=513536 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=2103851 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=505472 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=2103564 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=503168 peak_demand=66193 required_capacity=1.010025 + [2026-08-30 05:14:12] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 532.8 | throughput per GPU (TFLOP/s/GPU): 625.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.519414E+00 | loss scale: 1.0 | grad norm: 5.762 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131049 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=81920 expected_recv=127856 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131049 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=81920 expected_recv=127856 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=81920 expected_recv=127754 expected_overflow=0 exact_recv_per_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=129024 slack_slots=34816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=81920 expected_recv=127754 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=129152 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=81920 expected_recv=130911 expected_overflow=0 exact_recv_per_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=81920 expected_recv=128586 expected_overflow=0 exact_recv_per_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=81920 expected_recv=130911 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=81920 expected_recv=128586 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131133 expected_overflow=0 exact_recv_per_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=81920 expected_recv=128408 expected_overflow=0 exact_recv_per_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131133 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=81920 expected_recv=128408 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=129792 slack_slots=34048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=81920 expected_recv=129788 expected_overflow=0 exact_recv_per_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=81920 expected_recv=130875 expected_overflow=0 exact_recv_per_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=81920 expected_recv=129788 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=81920 expected_recv=130875 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=81920 expected_recv=128305 expected_overflow=0 exact_recv_per_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131105 expected_overflow=0 exact_recv_per_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131350 expected_overflow=0 exact_recv_per_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=81920 expected_recv=128305 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131105 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131350 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130980 expected_overflow=0 exact_recv_per_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131370 expected_overflow=0 exact_recv_per_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130980 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130988 expected_overflow=0 exact_recv_per_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131370 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130988 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130978 expected_overflow=0 exact_recv_per_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131056 expected_overflow=0 exact_recv_per_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131217 expected_overflow=0 exact_recv_per_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131217 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130978 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131056 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=81920 expected_recv=129603 expected_overflow=0 exact_recv_per_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=81920 expected_recv=129603 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=81920 expected_recv=132388 expected_overflow=0 exact_recv_per_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=81920 expected_recv=130709 expected_overflow=0 exact_recv_per_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=81920 expected_recv=132388 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=81920 expected_recv=130709 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131061 expected_overflow=0 exact_recv_per_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=81920 expected_recv=124885 expected_overflow=0 exact_recv_per_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131061 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=81920 expected_recv=124885 actual_valid_recv=125440 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=81920 expected_recv=129682 expected_overflow=0 exact_recv_per_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=125440 slack_slots=38400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=81920 expected_recv=130065 expected_overflow=0 exact_recv_per_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=81920 expected_recv=129682 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=81920 expected_recv=130487 expected_overflow=0 exact_recv_per_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=81920 expected_recv=130065 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=81920 expected_recv=130487 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=81920 expected_recv=128806 expected_overflow=0 exact_recv_per_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=81920 expected_recv=128806 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=130048 slack_slots=33792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=81920 expected_recv=129568 expected_overflow=0 exact_recv_per_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131066 expected_overflow=0 exact_recv_per_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=81920 expected_recv=129568 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131815 expected_overflow=0 exact_recv_per_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131066 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131815 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=81920 expected_recv=130911 expected_overflow=0 exact_recv_per_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131074 expected_overflow=0 exact_recv_per_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=81920 expected_recv=129372 expected_overflow=0 exact_recv_per_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=81920 expected_recv=130911 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131074 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=81920 expected_recv=129372 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=81920 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=81920 expected_recv=132387 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=81920 expected_recv=132388 expected_overflow=0 exact_recv_per_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=81920 expected_recv=130834 expected_overflow=0 exact_recv_per_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131082 expected_overflow=0 exact_recv_per_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=81920 expected_recv=132388 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=81920 expected_recv=130834 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131082 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=81920 expected_recv=130802 expected_overflow=0 exact_recv_per_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=81920 expected_recv=132380 expected_overflow=0 exact_recv_per_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=81920 expected_recv=127623 expected_overflow=0 exact_recv_per_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=81920 expected_recv=130802 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=81920 expected_recv=127623 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=81920 expected_recv=132380 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131083 expected_overflow=0 exact_recv_per_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=129152 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=81920 expected_recv=130410 expected_overflow=0 exact_recv_per_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=81920 expected_recv=126276 expected_overflow=0 exact_recv_per_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131083 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131050 expected_overflow=0 exact_recv_per_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=81920 expected_recv=130410 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=81920 expected_recv=126276 actual_valid_recv=127232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131050 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=127232 slack_slots=36608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131894 expected_overflow=0 exact_recv_per_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131894 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131796 expected_overflow=0 exact_recv_per_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131082 expected_overflow=0 exact_recv_per_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131084 expected_overflow=0 exact_recv_per_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131796 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131082 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131084 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131245 expected_overflow=0 exact_recv_per_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131834 expected_overflow=0 exact_recv_per_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=81920 expected_recv=127346 expected_overflow=0 exact_recv_per_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131834 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131245 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=81920 expected_recv=127346 actual_valid_recv=128768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131219 expected_overflow=0 exact_recv_per_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=128768 slack_slots=35072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=81920 expected_recv=127836 expected_overflow=0 exact_recv_per_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=81920 expected_recv=129344 expected_overflow=0 exact_recv_per_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131219 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=81920 expected_recv=127836 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=81920 expected_recv=129344 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=129152 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=81920 expected_recv=132380 expected_overflow=0 exact_recv_per_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=81920 expected_recv=129481 expected_overflow=0 exact_recv_per_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=81920 expected_recv=132380 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=81920 expected_recv=129481 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130849 expected_overflow=0 exact_recv_per_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131090 expected_overflow=0 exact_recv_per_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130849 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131090 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=81920 expected_recv=127578 expected_overflow=0 exact_recv_per_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130775 expected_overflow=0 exact_recv_per_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131133 expected_overflow=0 exact_recv_per_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=81920 expected_recv=127578 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130775 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131133 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=129024 slack_slots=34816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=81920 expected_recv=130193 expected_overflow=0 exact_recv_per_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=81920 expected_recv=128325 expected_overflow=0 exact_recv_per_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131106 expected_overflow=0 exact_recv_per_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=81920 expected_recv=130193 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=81920 expected_recv=128325 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131106 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131058 expected_overflow=0 exact_recv_per_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131088 expected_overflow=0 exact_recv_per_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131058 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131088 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=81920 expected_recv=129559 expected_overflow=0 exact_recv_per_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=81920 expected_recv=129559 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=81920 expected_recv=132040 expected_overflow=0 exact_recv_per_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=81920 expected_recv=132040 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=81920 expected_recv=129852 expected_overflow=0 exact_recv_per_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131138 expected_overflow=0 exact_recv_per_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=81920 expected_recv=129538 expected_overflow=0 exact_recv_per_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=81920 expected_recv=129852 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131138 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131339 expected_overflow=0 exact_recv_per_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=81920 expected_recv=129538 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131339 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131173 expected_overflow=0 exact_recv_per_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131173 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130985 expected_overflow=0 exact_recv_per_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130985 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131070 expected_overflow=0 exact_recv_per_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130972 expected_overflow=0 exact_recv_per_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131070 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131157 expected_overflow=0 exact_recv_per_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131157 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130972 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=81920 expected_recv=129428 expected_overflow=0 exact_recv_per_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131111 expected_overflow=0 exact_recv_per_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=81920 expected_recv=129428 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131111 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=81920 expected_recv=128452 expected_overflow=0 exact_recv_per_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=81920 expected_recv=128452 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=81920 expected_recv=130967 expected_overflow=0 exact_recv_per_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=81920 expected_recv=130753 expected_overflow=0 exact_recv_per_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=81920 expected_recv=130967 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=81920 expected_recv=130753 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[52197, 58460, 54170, 94550, 60590, 78808, 45545, 79968] avg=65536.0000 max=94550 max_rank=3 min=45545 min_rank=6 imbalance_ratio=1.442719 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[52197, 58460, 54170, 94550, 60590, 78808, 45545, 79968] avg=65536.0000 max=94550 max_rank=3 min=45545 min_rank=6 imbalance_ratio=1.442719 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=81920 expected_recv=129165 expected_overflow=0 exact_recv_per_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[52197, 58460, 54170, 94550, 60590, 78808, 45545, 79968] avg=65536.0000 max=94550 max_rank=3 min=45545 min_rank=6 imbalance_ratio=1.442719 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=81920 expected_recv=129165 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61663 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28358 max_load_reduction_pct=0.299926 moved_to_replicas=109066 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61663 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28358 max_load_reduction_pct=0.299926 moved_to_replicas=109066 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[52197, 58460, 54170, 94550, 60590, 78808, 45545, 79968] avg=65536.0000 max=94550 max_rank=3 min=45545 min_rank=6 imbalance_ratio=1.442719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[52197, 58460, 54170, 94550, 60590, 78808, 45545, 79968] avg=65536.0000 max=94550 max_rank=3 min=45545 min_rank=6 imbalance_ratio=1.442719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61663 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28358 max_load_reduction_pct=0.299926 moved_to_replicas=109066 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[52197, 58460, 54170, 94550, 60590, 78808, 45545, 79968] avg=65536.0000 max=94550 max_rank=3 min=45545 min_rank=6 imbalance_ratio=1.442719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[52197, 58460, 54170, 94550, 60590, 78808, 45545, 79968] avg=65536.0000 max=94550 max_rank=3 min=45545 min_rank=6 imbalance_ratio=1.442719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[65390, 62157, 31566, 103576, 32857, 54391, 65595, 108756] avg=65536.0000 max=108756 max_rank=7 min=31566 min_rank=2 imbalance_ratio=1.659485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61663 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28358 max_load_reduction_pct=0.299926 moved_to_replicas=109066 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61663 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28358 max_load_reduction_pct=0.299926 moved_to_replicas=109066 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[65390, 62157, 31566, 103576, 32857, 54391, 65595, 108756] avg=65536.0000 max=108756 max_rank=7 min=31566 min_rank=2 imbalance_ratio=1.659485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61663 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28358 max_load_reduction_pct=0.299926 moved_to_replicas=109066 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61663 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28358 max_load_reduction_pct=0.299926 moved_to_replicas=109066 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[52197, 58460, 54170, 94550, 60590, 78808, 45545, 79968] avg=65536.0000 max=94550 max_rank=3 min=45545 min_rank=6 imbalance_ratio=1.442719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[65390, 62157, 31566, 103576, 32857, 54391, 65595, 108756] avg=65536.0000 max=108756 max_rank=7 min=31566 min_rank=2 imbalance_ratio=1.659485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] avg=65536.0000 max=66193 max_rank=7 min=64222 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42563 max_load_reduction_pct=0.391362 moved_to_replicas=159828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] avg=65536.0000 max=66193 max_rank=7 min=64222 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42563 max_load_reduction_pct=0.391362 moved_to_replicas=159828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[65390, 62157, 31566, 103576, 32857, 54391, 65595, 108756] avg=65536.0000 max=108756 max_rank=7 min=31566 min_rank=2 imbalance_ratio=1.659485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[65390, 62157, 31566, 103576, 32857, 54391, 65595, 108756] avg=65536.0000 max=108756 max_rank=7 min=31566 min_rank=2 imbalance_ratio=1.659485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[66191, 66192, 61663, 66192, 65475, 66191, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=61663 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28358 max_load_reduction_pct=0.299926 moved_to_replicas=109066 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] avg=65536.0000 max=66193 max_rank=7 min=64222 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42563 max_load_reduction_pct=0.391362 moved_to_replicas=159828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[65390, 62157, 31566, 103576, 32857, 54391, 65595, 108756] avg=65536.0000 max=108756 max_rank=7 min=31566 min_rank=2 imbalance_ratio=1.659485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[65390, 62157, 31566, 103576, 32857, 54391, 65595, 108756] avg=65536.0000 max=108756 max_rank=7 min=31566 min_rank=2 imbalance_ratio=1.659485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32806, 98210, 65460, 77975, 86378, 65647, 65558, 32254] avg=65536.0000 max=98210 max_rank=1 min=32254 min_rank=7 imbalance_ratio=1.498566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] avg=65536.0000 max=66193 max_rank=7 min=64222 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42563 max_load_reduction_pct=0.391362 moved_to_replicas=159828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] avg=65536.0000 max=66193 max_rank=7 min=64222 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42563 max_load_reduction_pct=0.391362 moved_to_replicas=159828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32806, 98210, 65460, 77975, 86378, 65647, 65558, 32254] avg=65536.0000 max=98210 max_rank=1 min=32254 min_rank=7 imbalance_ratio=1.498566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] avg=65536.0000 max=66193 max_rank=7 min=64222 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42563 max_load_reduction_pct=0.391362 moved_to_replicas=159828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] avg=65536.0000 max=66193 max_rank=7 min=64222 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42563 max_load_reduction_pct=0.391362 moved_to_replicas=159828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32806, 98210, 65460, 77975, 86378, 65647, 65558, 32254] avg=65536.0000 max=98210 max_rank=1 min=32254 min_rank=7 imbalance_ratio=1.498566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[65390, 62157, 31566, 103576, 32857, 54391, 65595, 108756] avg=65536.0000 max=108756 max_rank=7 min=31566 min_rank=2 imbalance_ratio=1.659485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32018 max_load_reduction_pct=0.326016 moved_to_replicas=128203 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32018 max_load_reduction_pct=0.326016 moved_to_replicas=128203 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32806, 98210, 65460, 77975, 86378, 65647, 65558, 32254] avg=65536.0000 max=98210 max_rank=1 min=32254 min_rank=7 imbalance_ratio=1.498566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32806, 98210, 65460, 77975, 86378, 65647, 65558, 32254] avg=65536.0000 max=98210 max_rank=1 min=32254 min_rank=7 imbalance_ratio=1.498566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32018 max_load_reduction_pct=0.326016 moved_to_replicas=128203 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32806, 98210, 65460, 77975, 86378, 65647, 65558, 32254] avg=65536.0000 max=98210 max_rank=1 min=32254 min_rank=7 imbalance_ratio=1.498566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32806, 98210, 65460, 77975, 86378, 65647, 65558, 32254] avg=65536.0000 max=98210 max_rank=1 min=32254 min_rank=7 imbalance_ratio=1.498566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[65390, 66192, 64312, 66192, 66192, 64222, 65595, 66193] avg=65536.0000 max=66193 max_rank=7 min=64222 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=42563 max_load_reduction_pct=0.391362 moved_to_replicas=159828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65472, 130225, 65764, 32717, 65600, 33470, 65543, 65497] avg=65536.0000 max=130225 max_rank=1 min=32717 min_rank=3 imbalance_ratio=1.987076 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32018 max_load_reduction_pct=0.326016 moved_to_replicas=128203 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32018 max_load_reduction_pct=0.326016 moved_to_replicas=128203 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65472, 130225, 65764, 32717, 65600, 33470, 65543, 65497] avg=65536.0000 max=130225 max_rank=1 min=32717 min_rank=3 imbalance_ratio=1.987076 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32018 max_load_reduction_pct=0.326016 moved_to_replicas=128203 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32018 max_load_reduction_pct=0.326016 moved_to_replicas=128203 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65472, 130225, 65764, 32717, 65600, 33470, 65543, 65497] avg=65536.0000 max=130225 max_rank=1 min=32717 min_rank=3 imbalance_ratio=1.987076 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] avg=65536.0000 max=66192 max_rank=1 min=64744 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64033 max_load_reduction_pct=0.491711 moved_to_replicas=127821 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32806, 98210, 65460, 77975, 86378, 65647, 65558, 32254] avg=65536.0000 max=98210 max_rank=1 min=32254 min_rank=7 imbalance_ratio=1.498566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] avg=65536.0000 max=66192 max_rank=1 min=64744 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64033 max_load_reduction_pct=0.491711 moved_to_replicas=127821 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65472, 130225, 65764, 32717, 65600, 33470, 65543, 65497] avg=65536.0000 max=130225 max_rank=1 min=32717 min_rank=3 imbalance_ratio=1.987076 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65472, 130225, 65764, 32717, 65600, 33470, 65543, 65497] avg=65536.0000 max=130225 max_rank=1 min=32717 min_rank=3 imbalance_ratio=1.987076 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] avg=65536.0000 max=66192 max_rank=1 min=64744 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64033 max_load_reduction_pct=0.491711 moved_to_replicas=127821 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65472, 130225, 65764, 32717, 65600, 33470, 65543, 65497] avg=65536.0000 max=130225 max_rank=1 min=32717 min_rank=3 imbalance_ratio=1.987076 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65472, 130225, 65764, 32717, 65600, 33470, 65543, 65497] avg=65536.0000 max=130225 max_rank=1 min=32717 min_rank=3 imbalance_ratio=1.987076 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[64775, 66192, 65460, 66192, 66192, 65647, 65558, 64272] avg=65536.0000 max=66192 max_rank=1 min=64272 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32018 max_load_reduction_pct=0.326016 moved_to_replicas=128203 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32801, 88216, 27, 65524, 85766, 98286, 62830, 90838] avg=65536.0000 max=98286 max_rank=5 min=27 min_rank=2 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] avg=65536.0000 max=66192 max_rank=1 min=64744 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64033 max_load_reduction_pct=0.491711 moved_to_replicas=127821 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] avg=65536.0000 max=66192 max_rank=1 min=64744 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64033 max_load_reduction_pct=0.491711 moved_to_replicas=127821 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32801, 88216, 27, 65524, 85766, 98286, 62830, 90838] avg=65536.0000 max=98286 max_rank=5 min=27 min_rank=2 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] avg=65536.0000 max=66192 max_rank=1 min=64744 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64033 max_load_reduction_pct=0.491711 moved_to_replicas=127821 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] avg=65536.0000 max=66192 max_rank=1 min=64744 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64033 max_load_reduction_pct=0.491711 moved_to_replicas=127821 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] avg=65536.0000 max=66195 max_rank=0 min=62830 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=32091 max_load_reduction_pct=0.326506 moved_to_replicas=197433 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32801, 88216, 27, 65524, 85766, 98286, 62830, 90838] avg=65536.0000 max=98286 max_rank=5 min=27 min_rank=2 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65472, 130225, 65764, 32717, 65600, 33470, 65543, 65497] avg=65536.0000 max=130225 max_rank=1 min=32717 min_rank=3 imbalance_ratio=1.987076 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] avg=65536.0000 max=66195 max_rank=0 min=62830 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=32091 max_load_reduction_pct=0.326506 moved_to_replicas=197433 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32801, 88216, 27, 65524, 85766, 98286, 62830, 90838] avg=65536.0000 max=98286 max_rank=5 min=27 min_rank=2 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32801, 88216, 27, 65524, 85766, 98286, 62830, 90838] avg=65536.0000 max=98286 max_rank=5 min=27 min_rank=2 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32801, 88216, 27, 65524, 85766, 98286, 62830, 90838] avg=65536.0000 max=98286 max_rank=5 min=27 min_rank=2 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] avg=65536.0000 max=66195 max_rank=0 min=62830 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=32091 max_load_reduction_pct=0.326506 moved_to_replicas=197433 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32801, 88216, 27, 65524, 85766, 98286, 62830, 90838] avg=65536.0000 max=98286 max_rank=5 min=27 min_rank=2 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65472, 66192, 65764, 65476, 65600, 64744, 65543, 65497] avg=65536.0000 max=66192 max_rank=1 min=64744 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64033 max_load_reduction_pct=0.491711 moved_to_replicas=127821 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[32769, 32937, 65270, 98276, 131005, 98307, 42161, 23563] avg=65536.0000 max=131005 max_rank=4 min=23563 min_rank=7 imbalance_ratio=1.998978 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] avg=65536.0000 max=66195 max_rank=0 min=62830 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=32091 max_load_reduction_pct=0.326506 moved_to_replicas=197433 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] avg=65536.0000 max=66195 max_rank=0 min=62830 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=32091 max_load_reduction_pct=0.326506 moved_to_replicas=197433 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] avg=65536.0000 max=66195 max_rank=0 min=62830 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=32091 max_load_reduction_pct=0.326506 moved_to_replicas=197433 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[32769, 32937, 65270, 98276, 131005, 98307, 42161, 23563] avg=65536.0000 max=131005 max_rank=4 min=23563 min_rank=7 imbalance_ratio=1.998978 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] avg=65536.0000 max=66195 max_rank=0 min=62830 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=32091 max_load_reduction_pct=0.326506 moved_to_replicas=197433 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] avg=65536.0000 max=66193 max_rank=6 min=64375 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=64812 max_load_reduction_pct=0.494729 moved_to_replicas=258040 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[32769, 32937, 65270, 98276, 131005, 98307, 42161, 23563] avg=65536.0000 max=131005 max_rank=4 min=23563 min_rank=7 imbalance_ratio=1.998978 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[32769, 32937, 65270, 98276, 131005, 98307, 42161, 23563] avg=65536.0000 max=131005 max_rank=4 min=23563 min_rank=7 imbalance_ratio=1.998978 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32801, 88216, 27, 65524, 85766, 98286, 62830, 90838] avg=65536.0000 max=98286 max_rank=5 min=27 min_rank=2 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] avg=65536.0000 max=66193 max_rank=6 min=64375 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=64812 max_load_reduction_pct=0.494729 moved_to_replicas=258040 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[32769, 32937, 65270, 98276, 131005, 98307, 42161, 23563] avg=65536.0000 max=131005 max_rank=4 min=23563 min_rank=7 imbalance_ratio=1.998978 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[32769, 32937, 65270, 98276, 131005, 98307, 42161, 23563] avg=65536.0000 max=131005 max_rank=4 min=23563 min_rank=7 imbalance_ratio=1.998978 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] avg=65536.0000 max=66193 max_rank=6 min=64375 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=64812 max_load_reduction_pct=0.494729 moved_to_replicas=258040 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[32769, 32937, 65270, 98276, 131005, 98307, 42161, 23563] avg=65536.0000 max=131005 max_rank=4 min=23563 min_rank=7 imbalance_ratio=1.998978 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 32782, 32676, 78764, 150820, 32852] avg=65536.0000 max=150820 max_rank=6 min=32676 min_rank=4 imbalance_ratio=2.301331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] avg=65536.0000 max=66193 max_rank=6 min=64375 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=64812 max_load_reduction_pct=0.494729 moved_to_replicas=258040 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[66195, 66191, 64974, 65524, 66191, 66192, 62830, 66191] avg=65536.0000 max=66195 max_rank=0 min=62830 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=32091 max_load_reduction_pct=0.326506 moved_to_replicas=197433 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] avg=65536.0000 max=66193 max_rank=6 min=64375 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=64812 max_load_reduction_pct=0.494729 moved_to_replicas=258040 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] avg=65536.0000 max=66193 max_rank=6 min=64375 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=64812 max_load_reduction_pct=0.494729 moved_to_replicas=258040 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 32782, 32676, 78764, 150820, 32852] avg=65536.0000 max=150820 max_rank=6 min=32676 min_rank=4 imbalance_ratio=2.301331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] avg=65536.0000 max=66193 max_rank=6 min=64375 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=64812 max_load_reduction_pct=0.494729 moved_to_replicas=258040 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] avg=65536.0000 max=66193 max_rank=5 min=64531 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=84627 max_load_reduction_pct=0.561113 moved_to_replicas=194759 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 32782, 32676, 78764, 150820, 32852] avg=65536.0000 max=150820 max_rank=6 min=32676 min_rank=4 imbalance_ratio=2.301331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 32782, 32676, 78764, 150820, 32852] avg=65536.0000 max=150820 max_rank=6 min=32676 min_rank=4 imbalance_ratio=2.301331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] avg=65536.0000 max=66193 max_rank=5 min=64531 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=84627 max_load_reduction_pct=0.561113 moved_to_replicas=194759 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 32782, 32676, 78764, 150820, 32852] avg=65536.0000 max=150820 max_rank=6 min=32676 min_rank=4 imbalance_ratio=2.301331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[32769, 32937, 65270, 98276, 131005, 98307, 42161, 23563] avg=65536.0000 max=131005 max_rank=4 min=23563 min_rank=7 imbalance_ratio=1.998978 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 32782, 32676, 78764, 150820, 32852] avg=65536.0000 max=150820 max_rank=6 min=32676 min_rank=4 imbalance_ratio=2.301331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] avg=65536.0000 max=66193 max_rank=5 min=64531 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=84627 max_load_reduction_pct=0.561113 moved_to_replicas=194759 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 32782, 32676, 78764, 150820, 32852] avg=65536.0000 max=150820 max_rank=6 min=32676 min_rank=4 imbalance_ratio=2.301331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[75048, 33518, 65542, 186627, 32625, 28, 32821, 98079] avg=65536.0000 max=186627 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.847702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] avg=65536.0000 max=66193 max_rank=5 min=64531 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=84627 max_load_reduction_pct=0.561113 moved_to_replicas=194759 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] avg=65536.0000 max=66193 max_rank=5 min=64531 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=84627 max_load_reduction_pct=0.561113 moved_to_replicas=194759 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] avg=65536.0000 max=66193 max_rank=5 min=64531 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=84627 max_load_reduction_pct=0.561113 moved_to_replicas=194759 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[64822, 65052, 65270, 66192, 66192, 66192, 66193, 64375] avg=65536.0000 max=66193 max_rank=6 min=64375 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=64812 max_load_reduction_pct=0.494729 moved_to_replicas=258040 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[75048, 33518, 65542, 186627, 32625, 28, 32821, 98079] avg=65536.0000 max=186627 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.847702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] avg=65536.0000 max=66193 max_rank=5 min=64531 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=84627 max_load_reduction_pct=0.561113 moved_to_replicas=194759 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] avg=65536.0000 max=66195 max_rank=0 min=63835 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=120432 max_load_reduction_pct=0.645309 moved_to_replicas=322260 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[75048, 33518, 65542, 186627, 32625, 28, 32821, 98079] avg=65536.0000 max=186627 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.847702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[75048, 33518, 65542, 186627, 32625, 28, 32821, 98079] avg=65536.0000 max=186627 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.847702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[75048, 33518, 65542, 186627, 32625, 28, 32821, 98079] avg=65536.0000 max=186627 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.847702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] avg=65536.0000 max=66195 max_rank=0 min=63835 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=120432 max_load_reduction_pct=0.645309 moved_to_replicas=322260 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[75048, 33518, 65542, 186627, 32625, 28, 32821, 98079] avg=65536.0000 max=186627 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.847702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 32782, 32676, 78764, 150820, 32852] avg=65536.0000 max=150820 max_rank=6 min=32676 min_rank=4 imbalance_ratio=2.301331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] avg=65536.0000 max=66195 max_rank=0 min=63835 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=120432 max_load_reduction_pct=0.645309 moved_to_replicas=322260 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[75048, 33518, 65542, 186627, 32625, 28, 32821, 98079] avg=65536.0000 max=186627 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.847702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[32828, 56325, 67, 98892, 106536, 32855, 130982, 65803] avg=65536.0000 max=130982 max_rank=6 min=67 min_rank=2 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] avg=65536.0000 max=66195 max_rank=0 min=63835 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=120432 max_load_reduction_pct=0.645309 moved_to_replicas=322260 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] avg=65536.0000 max=66195 max_rank=0 min=63835 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=120432 max_load_reduction_pct=0.645309 moved_to_replicas=322260 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] avg=65536.0000 max=66195 max_rank=0 min=63835 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=120432 max_load_reduction_pct=0.645309 moved_to_replicas=322260 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[64950, 65533, 65911, 65541, 65436, 66193, 66193, 64531] avg=65536.0000 max=66193 max_rank=5 min=64531 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=84627 max_load_reduction_pct=0.561113 moved_to_replicas=194759 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] avg=65536.0000 max=66195 max_rank=0 min=63835 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=120432 max_load_reduction_pct=0.645309 moved_to_replicas=322260 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] avg=65536.0000 max=66194 max_rank=5 min=63271 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64788 max_load_reduction_pct=0.494633 moved_to_replicas=272923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[32828, 56325, 67, 98892, 106536, 32855, 130982, 65803] avg=65536.0000 max=130982 max_rank=6 min=67 min_rank=2 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[32828, 56325, 67, 98892, 106536, 32855, 130982, 65803] avg=65536.0000 max=130982 max_rank=6 min=67 min_rank=2 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[32828, 56325, 67, 98892, 106536, 32855, 130982, 65803] avg=65536.0000 max=130982 max_rank=6 min=67 min_rank=2 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[32828, 56325, 67, 98892, 106536, 32855, 130982, 65803] avg=65536.0000 max=130982 max_rank=6 min=67 min_rank=2 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] avg=65536.0000 max=66194 max_rank=5 min=63271 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64788 max_load_reduction_pct=0.494633 moved_to_replicas=272923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[32828, 56325, 67, 98892, 106536, 32855, 130982, 65803] avg=65536.0000 max=130982 max_rank=6 min=67 min_rank=2 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] avg=65536.0000 max=66194 max_rank=5 min=63271 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64788 max_load_reduction_pct=0.494633 moved_to_replicas=272923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[75048, 33518, 65542, 186627, 32625, 28, 32821, 98079] avg=65536.0000 max=186627 max_rank=3 min=28 min_rank=5 imbalance_ratio=2.847702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[117675, 65542, 32787, 32787, 66418, 65663, 32883, 110533] avg=65536.0000 max=117675 max_rank=0 min=32787 min_rank=2 imbalance_ratio=1.795578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[32828, 56325, 67, 98892, 106536, 32855, 130982, 65803] avg=65536.0000 max=130982 max_rank=6 min=67 min_rank=2 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] avg=65536.0000 max=66194 max_rank=5 min=63271 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64788 max_load_reduction_pct=0.494633 moved_to_replicas=272923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] avg=65536.0000 max=66194 max_rank=5 min=63271 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64788 max_load_reduction_pct=0.494633 moved_to_replicas=272923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] avg=65536.0000 max=66194 max_rank=5 min=63271 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64788 max_load_reduction_pct=0.494633 moved_to_replicas=272923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[66195, 65405, 65542, 66190, 65383, 65546, 63835, 66192] avg=65536.0000 max=66195 max_rank=0 min=63835 min_rank=6 imbalance_ratio=1.010056 max_load_reduction=120432 max_load_reduction_pct=0.645309 moved_to_replicas=322260 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] avg=65536.0000 max=66192 max_rank=0 min=63318 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=51483 max_load_reduction_pct=0.437502 moved_to_replicas=192817 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[117675, 65542, 32787, 32787, 66418, 65663, 32883, 110533] avg=65536.0000 max=117675 max_rank=0 min=32787 min_rank=2 imbalance_ratio=1.795578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] avg=65536.0000 max=66194 max_rank=5 min=63271 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64788 max_load_reduction_pct=0.494633 moved_to_replicas=272923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[117675, 65542, 32787, 32787, 66418, 65663, 32883, 110533] avg=65536.0000 max=117675 max_rank=0 min=32787 min_rank=2 imbalance_ratio=1.795578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[117675, 65542, 32787, 32787, 66418, 65663, 32883, 110533] avg=65536.0000 max=117675 max_rank=0 min=32787 min_rank=2 imbalance_ratio=1.795578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[117675, 65542, 32787, 32787, 66418, 65663, 32883, 110533] avg=65536.0000 max=117675 max_rank=0 min=32787 min_rank=2 imbalance_ratio=1.795578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] avg=65536.0000 max=66192 max_rank=0 min=63318 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=51483 max_load_reduction_pct=0.437502 moved_to_replicas=192817 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[117675, 65542, 32787, 32787, 66418, 65663, 32883, 110533] avg=65536.0000 max=117675 max_rank=0 min=32787 min_rank=2 imbalance_ratio=1.795578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] avg=65536.0000 max=66192 max_rank=0 min=63318 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=51483 max_load_reduction_pct=0.437502 moved_to_replicas=192817 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[32828, 56325, 67, 98892, 106536, 32855, 130982, 65803] avg=65536.0000 max=130982 max_rank=6 min=67 min_rank=2 imbalance_ratio=1.998627 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] avg=65536.0000 max=66192 max_rank=0 min=63318 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=51483 max_load_reduction_pct=0.437502 moved_to_replicas=192817 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[117675, 65542, 32787, 32787, 66418, 65663, 32883, 110533] avg=65536.0000 max=117675 max_rank=0 min=32787 min_rank=2 imbalance_ratio=1.795578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[98015, 9995, 32845, 98267, 98270, 76932, 77311, 32653] avg=65536.0000 max=98270 max_rank=4 min=9995 min_rank=1 imbalance_ratio=1.499481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] avg=65536.0000 max=66192 max_rank=0 min=63318 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=51483 max_load_reduction_pct=0.437502 moved_to_replicas=192817 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] avg=65536.0000 max=66192 max_rank=0 min=63318 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=51483 max_load_reduction_pct=0.437502 moved_to_replicas=192817 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[98015, 9995, 32845, 98267, 98270, 76932, 77311, 32653] avg=65536.0000 max=98270 max_rank=4 min=9995 min_rank=1 imbalance_ratio=1.499481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65588, 63271, 64857, 66192, 66191, 66194, 66192, 65803] avg=65536.0000 max=66194 max_rank=5 min=63271 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=64788 max_load_reduction_pct=0.494633 moved_to_replicas=272923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] avg=65536.0000 max=66192 max_rank=0 min=63318 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=51483 max_load_reduction_pct=0.437502 moved_to_replicas=192817 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] avg=65536.0000 max=66192 max_rank=0 min=63936 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32078 max_load_reduction_pct=0.326427 moved_to_replicas=240348 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[98015, 9995, 32845, 98267, 98270, 76932, 77311, 32653] avg=65536.0000 max=98270 max_rank=4 min=9995 min_rank=1 imbalance_ratio=1.499481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[98015, 9995, 32845, 98267, 98270, 76932, 77311, 32653] avg=65536.0000 max=98270 max_rank=4 min=9995 min_rank=1 imbalance_ratio=1.499481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[98015, 9995, 32845, 98267, 98270, 76932, 77311, 32653] avg=65536.0000 max=98270 max_rank=4 min=9995 min_rank=1 imbalance_ratio=1.499481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] avg=65536.0000 max=66192 max_rank=0 min=63936 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32078 max_load_reduction_pct=0.326427 moved_to_replicas=240348 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[98015, 9995, 32845, 98267, 98270, 76932, 77311, 32653] avg=65536.0000 max=98270 max_rank=4 min=9995 min_rank=1 imbalance_ratio=1.499481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] avg=65536.0000 max=66192 max_rank=0 min=63936 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32078 max_load_reduction_pct=0.326427 moved_to_replicas=240348 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] avg=65536.0000 max=66192 max_rank=0 min=63936 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32078 max_load_reduction_pct=0.326427 moved_to_replicas=240348 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[117675, 65542, 32787, 32787, 66418, 65663, 32883, 110533] avg=65536.0000 max=117675 max_rank=0 min=32787 min_rank=2 imbalance_ratio=1.795578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[98015, 9995, 32845, 98267, 98270, 76932, 77311, 32653] avg=65536.0000 max=98270 max_rank=4 min=9995 min_rank=1 imbalance_ratio=1.499481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[65331, 73650, 65556, 130499, 29, 122518, 1123, 65582] avg=65536.0000 max=130499 max_rank=3 min=29 min_rank=4 imbalance_ratio=1.991257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] avg=65536.0000 max=66192 max_rank=0 min=63936 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32078 max_load_reduction_pct=0.326427 moved_to_replicas=240348 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] avg=65536.0000 max=66192 max_rank=0 min=63936 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32078 max_load_reduction_pct=0.326427 moved_to_replicas=240348 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[65331, 73650, 65556, 130499, 29, 122518, 1123, 65582] avg=65536.0000 max=130499 max_rank=3 min=29 min_rank=4 imbalance_ratio=1.991257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[66192, 65542, 65547, 63318, 66192, 65663, 65642, 66192] avg=65536.0000 max=66192 max_rank=0 min=63318 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=51483 max_load_reduction_pct=0.437502 moved_to_replicas=192817 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] avg=65536.0000 max=66192 max_rank=0 min=63936 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32078 max_load_reduction_pct=0.326427 moved_to_replicas=240348 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] avg=65536.0000 max=66194 max_rank=5 min=63810 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64305 max_load_reduction_pct=0.492762 moved_to_replicas=256606 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[65331, 73650, 65556, 130499, 29, 122518, 1123, 65582] avg=65536.0000 max=130499 max_rank=3 min=29 min_rank=4 imbalance_ratio=1.991257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[65331, 73650, 65556, 130499, 29, 122518, 1123, 65582] avg=65536.0000 max=130499 max_rank=3 min=29 min_rank=4 imbalance_ratio=1.991257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[65331, 73650, 65556, 130499, 29, 122518, 1123, 65582] avg=65536.0000 max=130499 max_rank=3 min=29 min_rank=4 imbalance_ratio=1.991257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[65331, 73650, 65556, 130499, 29, 122518, 1123, 65582] avg=65536.0000 max=130499 max_rank=3 min=29 min_rank=4 imbalance_ratio=1.991257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] avg=65536.0000 max=66194 max_rank=5 min=63810 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64305 max_load_reduction_pct=0.492762 moved_to_replicas=256606 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] avg=65536.0000 max=66194 max_rank=5 min=63810 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64305 max_load_reduction_pct=0.492762 moved_to_replicas=256606 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] avg=65536.0000 max=66194 max_rank=5 min=63810 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64305 max_load_reduction_pct=0.492762 moved_to_replicas=256606 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] avg=65536.0000 max=66194 max_rank=5 min=63810 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64305 max_load_reduction_pct=0.492762 moved_to_replicas=256606 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[98015, 9995, 32845, 98267, 98270, 76932, 77311, 32653] avg=65536.0000 max=98270 max_rank=4 min=9995 min_rank=1 imbalance_ratio=1.499481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[65331, 73650, 65556, 130499, 29, 122518, 1123, 65582] avg=65536.0000 max=130499 max_rank=3 min=29 min_rank=4 imbalance_ratio=1.991257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[65070, 32441, 65571, 72688, 65523, 66396, 91055, 65544] avg=65536.0000 max=91055 max_rank=6 min=32441 min_rank=1 imbalance_ratio=1.389389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] avg=65536.0000 max=66194 max_rank=5 min=63810 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64305 max_load_reduction_pct=0.492762 moved_to_replicas=256606 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[65070, 32441, 65571, 72688, 65523, 66396, 91055, 65544] avg=65536.0000 max=91055 max_rank=6 min=32441 min_rank=1 imbalance_ratio=1.389389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[66192, 63936, 64668, 66192, 66192, 66191, 66189, 64728] avg=65536.0000 max=66192 max_rank=0 min=63936 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32078 max_load_reduction_pct=0.326427 moved_to_replicas=240348 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] avg=65536.0000 max=66194 max_rank=5 min=63810 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64305 max_load_reduction_pct=0.492762 moved_to_replicas=256606 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] avg=65536.0000 max=66193 max_rank=3 min=64002 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=24862 max_load_reduction_pct=0.273044 moved_to_replicas=63316 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[65070, 32441, 65571, 72688, 65523, 66396, 91055, 65544] avg=65536.0000 max=91055 max_rank=6 min=32441 min_rank=1 imbalance_ratio=1.389389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[65070, 32441, 65571, 72688, 65523, 66396, 91055, 65544] avg=65536.0000 max=91055 max_rank=6 min=32441 min_rank=1 imbalance_ratio=1.389389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[65070, 32441, 65571, 72688, 65523, 66396, 91055, 65544] avg=65536.0000 max=91055 max_rank=6 min=32441 min_rank=1 imbalance_ratio=1.389389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] avg=65536.0000 max=66193 max_rank=3 min=64002 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=24862 max_load_reduction_pct=0.273044 moved_to_replicas=63316 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[65070, 32441, 65571, 72688, 65523, 66396, 91055, 65544] avg=65536.0000 max=91055 max_rank=6 min=32441 min_rank=1 imbalance_ratio=1.389389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] avg=65536.0000 max=66193 max_rank=3 min=64002 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=24862 max_load_reduction_pct=0.273044 moved_to_replicas=63316 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] avg=65536.0000 max=66193 max_rank=3 min=64002 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=24862 max_load_reduction_pct=0.273044 moved_to_replicas=63316 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[65331, 73650, 65556, 130499, 29, 122518, 1123, 65582] avg=65536.0000 max=130499 max_rank=3 min=29 min_rank=4 imbalance_ratio=1.991257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[65070, 32441, 65571, 72688, 65523, 66396, 91055, 65544] avg=65536.0000 max=91055 max_rank=6 min=32441 min_rank=1 imbalance_ratio=1.389389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] avg=65536.0000 max=66193 max_rank=3 min=64002 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=24862 max_load_reduction_pct=0.273044 moved_to_replicas=63316 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[130919, 32798, 114644, 32928, 64833, 55744, 65575, 26847] avg=65536.0000 max=130919 max_rank=0 min=26847 min_rank=7 imbalance_ratio=1.997665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] avg=65536.0000 max=66193 max_rank=3 min=64002 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=24862 max_load_reduction_pct=0.273044 moved_to_replicas=63316 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[130919, 32798, 114644, 32928, 64833, 55744, 65575, 26847] avg=65536.0000 max=130919 max_rank=0 min=26847 min_rank=7 imbalance_ratio=1.997665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[130919, 32798, 114644, 32928, 64833, 55744, 65575, 26847] avg=65536.0000 max=130919 max_rank=0 min=26847 min_rank=7 imbalance_ratio=1.997665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[65331, 66192, 65556, 66192, 63810, 66194, 65431, 65582] avg=65536.0000 max=66194 max_rank=5 min=63810 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64305 max_load_reduction_pct=0.492762 moved_to_replicas=256606 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] avg=65536.0000 max=66193 max_rank=3 min=64002 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=24862 max_load_reduction_pct=0.273044 moved_to_replicas=63316 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[130919, 32798, 114644, 32928, 64833, 55744, 65575, 26847] avg=65536.0000 max=130919 max_rank=0 min=26847 min_rank=7 imbalance_ratio=1.997665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] avg=65536.0000 max=66192 max_rank=0 min=64770 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64727 max_load_reduction_pct=0.494405 moved_to_replicas=226862 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[130919, 32798, 114644, 32928, 64833, 55744, 65575, 26847] avg=65536.0000 max=130919 max_rank=0 min=26847 min_rank=7 imbalance_ratio=1.997665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] avg=65536.0000 max=66192 max_rank=0 min=64770 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64727 max_load_reduction_pct=0.494405 moved_to_replicas=226862 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[130919, 32798, 114644, 32928, 64833, 55744, 65575, 26847] avg=65536.0000 max=130919 max_rank=0 min=26847 min_rank=7 imbalance_ratio=1.997665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] avg=65536.0000 max=66192 max_rank=0 min=64770 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64727 max_load_reduction_pct=0.494405 moved_to_replicas=226862 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] avg=65536.0000 max=66192 max_rank=0 min=64770 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64727 max_load_reduction_pct=0.494405 moved_to_replicas=226862 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] avg=65536.0000 max=66192 max_rank=0 min=64770 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64727 max_load_reduction_pct=0.494405 moved_to_replicas=226862 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[65070, 32441, 65571, 72688, 65523, 66396, 91055, 65544] avg=65536.0000 max=91055 max_rank=6 min=32441 min_rank=1 imbalance_ratio=1.389389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[69810, 32843, 65449, 32842, 159399, 32961, 65535, 65449] avg=65536.0000 max=159399 max_rank=4 min=32842 min_rank=3 imbalance_ratio=2.432236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[130919, 32798, 114644, 32928, 64833, 55744, 65575, 26847] avg=65536.0000 max=130919 max_rank=0 min=26847 min_rank=7 imbalance_ratio=1.997665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] avg=65536.0000 max=66192 max_rank=0 min=64770 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64727 max_load_reduction_pct=0.494405 moved_to_replicas=226862 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[69810, 32843, 65449, 32842, 159399, 32961, 65535, 65449] avg=65536.0000 max=159399 max_rank=4 min=32842 min_rank=3 imbalance_ratio=2.432236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[69810, 32843, 65449, 32842, 159399, 32961, 65535, 65449] avg=65536.0000 max=159399 max_rank=4 min=32842 min_rank=3 imbalance_ratio=2.432236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[69810, 32843, 65449, 32842, 159399, 32961, 65535, 65449] avg=65536.0000 max=159399 max_rank=4 min=32842 min_rank=3 imbalance_ratio=2.432236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[65070, 64002, 65571, 66193, 65523, 66192, 66193, 65544] avg=65536.0000 max=66193 max_rank=3 min=64002 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=24862 max_load_reduction_pct=0.273044 moved_to_replicas=63316 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] avg=65536.0000 max=66192 max_rank=0 min=64267 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93207 max_load_reduction_pct=0.584740 moved_to_replicas=193544 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] avg=65536.0000 max=66192 max_rank=0 min=64770 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64727 max_load_reduction_pct=0.494405 moved_to_replicas=226862 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[69810, 32843, 65449, 32842, 159399, 32961, 65535, 65449] avg=65536.0000 max=159399 max_rank=4 min=32842 min_rank=3 imbalance_ratio=2.432236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] avg=65536.0000 max=66192 max_rank=0 min=64267 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93207 max_load_reduction_pct=0.584740 moved_to_replicas=193544 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[69810, 32843, 65449, 32842, 159399, 32961, 65535, 65449] avg=65536.0000 max=159399 max_rank=4 min=32842 min_rank=3 imbalance_ratio=2.432236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] avg=65536.0000 max=66192 max_rank=0 min=64267 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93207 max_load_reduction_pct=0.584740 moved_to_replicas=193544 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] avg=65536.0000 max=66192 max_rank=0 min=64267 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93207 max_load_reduction_pct=0.584740 moved_to_replicas=193544 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] avg=65536.0000 max=66192 max_rank=0 min=64267 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93207 max_load_reduction_pct=0.584740 moved_to_replicas=193544 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[130919, 32798, 114644, 32928, 64833, 55744, 65575, 26847] avg=65536.0000 max=130919 max_rank=0 min=26847 min_rank=7 imbalance_ratio=1.997665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] avg=65536.0000 max=66192 max_rank=0 min=64267 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93207 max_load_reduction_pct=0.584740 moved_to_replicas=193544 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[69810, 32843, 65449, 32842, 159399, 32961, 65535, 65449] avg=65536.0000 max=159399 max_rank=4 min=32842 min_rank=3 imbalance_ratio=2.432236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[33166, 65562, 98806, 65283, 65492, 168, 97584, 98227] avg=65536.0000 max=98806 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.507660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[33166, 65562, 98806, 65283, 65492, 168, 97584, 98227] avg=65536.0000 max=98806 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.507660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[33166, 65562, 98806, 65283, 65492, 168, 97584, 98227] avg=65536.0000 max=98806 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.507660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[33166, 65562, 98806, 65283, 65492, 168, 97584, 98227] avg=65536.0000 max=98806 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.507660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[66192, 64770, 66192, 65686, 64833, 66192, 65575, 64848] avg=65536.0000 max=66192 max_rank=0 min=64770 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64727 max_load_reduction_pct=0.494405 moved_to_replicas=226862 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] avg=65536.0000 max=66192 max_rank=0 min=64267 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93207 max_load_reduction_pct=0.584740 moved_to_replicas=193544 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32614 max_load_reduction_pct=0.330081 moved_to_replicas=192299 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[33166, 65562, 98806, 65283, 65492, 168, 97584, 98227] avg=65536.0000 max=98806 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.507660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[33166, 65562, 98806, 65283, 65492, 168, 97584, 98227] avg=65536.0000 max=98806 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.507660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32614 max_load_reduction_pct=0.330081 moved_to_replicas=192299 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32614 max_load_reduction_pct=0.330081 moved_to_replicas=192299 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32614 max_load_reduction_pct=0.330081 moved_to_replicas=192299 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32614 max_load_reduction_pct=0.330081 moved_to_replicas=192299 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[69810, 32843, 65449, 32842, 159399, 32961, 65535, 65449] avg=65536.0000 max=159399 max_rank=4 min=32842 min_rank=3 imbalance_ratio=2.432236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32614 max_load_reduction_pct=0.330081 moved_to_replicas=192299 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[33166, 65562, 98806, 65283, 65492, 168, 97584, 98227] avg=65536.0000 max=98806 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.507660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[66192, 65602, 65449, 65602, 66192, 64267, 65535, 65449] avg=65536.0000 max=66192 max_rank=0 min=64267 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=93207 max_load_reduction_pct=0.584740 moved_to_replicas=193544 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32614 max_load_reduction_pct=0.330081 moved_to_replicas=192299 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[33166, 65562, 98806, 65283, 65492, 168, 97584, 98227] avg=65536.0000 max=98806 max_rank=2 min=168 min_rank=5 imbalance_ratio=1.507660 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[64558, 65562, 66192, 65283, 65492, 64817, 66192, 66192] avg=65536.0000 max=66192 max_rank=2 min=64558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32614 max_load_reduction_pct=0.330081 moved_to_replicas=192299 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=2098505 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=505856 peak_demand=66195 required_capacity=1.010056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=2096632 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506880 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=2090292 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=513280 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=2092207 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=510976 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=2098662 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=504192 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=2095947 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507520 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=2105168 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=498816 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=2099803 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=502784 peak_demand=66194 required_capacity=1.010040 + [2026-08-30 05:14:12] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 539.1 | throughput per GPU (TFLOP/s/GPU): 618.5 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.273413E+00 | loss scale: 1.0 | grad norm: 5.594 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=81920 expected_recv=129920 expected_overflow=0 exact_recv_per_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=81920 expected_recv=127363 expected_overflow=0 exact_recv_per_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=81920 expected_recv=129814 expected_overflow=0 exact_recv_per_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=81920 expected_recv=129920 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=81920 expected_recv=127363 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=81920 expected_recv=129814 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131940 expected_overflow=0 exact_recv_per_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=128384 slack_slots=35456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131940 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131008 expected_overflow=0 exact_recv_per_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131008 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=81920 expected_recv=130070 expected_overflow=0 exact_recv_per_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=81920 expected_recv=130070 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=81920 expected_recv=127895 expected_overflow=0 exact_recv_per_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=81920 expected_recv=130067 expected_overflow=0 exact_recv_per_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=81920 expected_recv=127895 actual_valid_recv=129024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=81920 expected_recv=130067 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=81920 expected_recv=129394 expected_overflow=0 exact_recv_per_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=129024 slack_slots=34816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=134528 slack_slots=29312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=81920 expected_recv=129394 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131086 expected_overflow=0 exact_recv_per_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131119 expected_overflow=0 exact_recv_per_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=81920 expected_recv=127441 expected_overflow=0 exact_recv_per_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131086 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131119 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130896 expected_overflow=0 exact_recv_per_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=81920 expected_recv=127441 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130896 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=128640 slack_slots=35200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=134272 slack_slots=29568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130940 expected_overflow=0 exact_recv_per_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130940 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=81920 expected_recv=128240 expected_overflow=0 exact_recv_per_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=81920 expected_recv=129645 expected_overflow=0 exact_recv_per_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=81920 expected_recv=131703 expected_overflow=0 exact_recv_per_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=81920 expected_recv=130005 expected_overflow=0 exact_recv_per_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=81920 expected_recv=128240 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=81920 expected_recv=129645 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=81920 expected_recv=131703 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=81920 expected_recv=130005 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=81920 expected_recv=129212 expected_overflow=0 exact_recv_per_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131092 expected_overflow=0 exact_recv_per_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=81920 expected_recv=129212 actual_valid_recv=130688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131092 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=81920 expected_recv=129459 expected_overflow=0 exact_recv_per_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=130688 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=81920 expected_recv=129459 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131656 expected_overflow=0 exact_recv_per_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131726 expected_overflow=0 exact_recv_per_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131656 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131726 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=81920 expected_recv=130642 expected_overflow=0 exact_recv_per_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=81920 expected_recv=129697 expected_overflow=0 exact_recv_per_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=81920 expected_recv=130642 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=81920 expected_recv=129697 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=81920 expected_recv=126975 expected_overflow=0 exact_recv_per_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=81920 expected_recv=131520 expected_overflow=0 exact_recv_per_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=81920 expected_recv=126975 actual_valid_recv=128256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=81920 expected_recv=131520 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=128256 slack_slots=35584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=81920 expected_recv=129402 expected_overflow=0 exact_recv_per_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=81920 expected_recv=129402 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=81920 expected_recv=125733 expected_overflow=0 exact_recv_per_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=81920 expected_recv=125733 actual_valid_recv=127104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=127104 slack_slots=36736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=81920 expected_recv=125867 expected_overflow=0 exact_recv_per_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=81920 expected_recv=125867 actual_valid_recv=127488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=127488 slack_slots=36352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=81920 expected_recv=130918 expected_overflow=0 exact_recv_per_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131164 expected_overflow=0 exact_recv_per_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=81920 expected_recv=129552 expected_overflow=0 exact_recv_per_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131092 expected_overflow=0 exact_recv_per_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=81920 expected_recv=130918 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131164 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=81920 expected_recv=129552 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=81920 expected_recv=129155 expected_overflow=0 exact_recv_per_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131092 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=81920 expected_recv=129037 expected_overflow=0 exact_recv_per_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=81920 expected_recv=129155 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=81920 expected_recv=129037 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=81920 expected_recv=132170 expected_overflow=0 exact_recv_per_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=81920 expected_recv=132170 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131510 expected_overflow=0 exact_recv_per_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131045 expected_overflow=0 exact_recv_per_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131510 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=81920 expected_recv=129934 expected_overflow=0 exact_recv_per_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131045 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=81920 expected_recv=129934 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=81920 expected_recv=129929 expected_overflow=0 exact_recv_per_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=81920 expected_recv=129929 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=81920 expected_recv=128131 expected_overflow=0 exact_recv_per_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=81920 expected_recv=128131 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=81920 expected_recv=130669 expected_overflow=0 exact_recv_per_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=81920 expected_recv=130938 expected_overflow=0 exact_recv_per_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=81920 expected_recv=130669 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=81920 expected_recv=130938 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=81920 expected_recv=125049 expected_overflow=0 exact_recv_per_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=81920 expected_recv=130812 expected_overflow=0 exact_recv_per_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=81920 expected_recv=125049 actual_valid_recv=126464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=81920 expected_recv=130812 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=126464 slack_slots=37376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=81920 expected_recv=132366 expected_overflow=0 exact_recv_per_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=81920 expected_recv=132366 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=81920 expected_recv=129113 expected_overflow=0 exact_recv_per_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=81920 expected_recv=129700 expected_overflow=0 exact_recv_per_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=81920 expected_recv=124864 expected_overflow=0 exact_recv_per_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=81920 expected_recv=129113 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=81920 expected_recv=129700 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=81920 expected_recv=129433 expected_overflow=0 exact_recv_per_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=81920 expected_recv=124864 actual_valid_recv=126592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=81920 expected_recv=129433 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=126592 slack_slots=37248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=81920 expected_recv=132133 expected_overflow=0 exact_recv_per_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=81920 expected_recv=132133 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131000 expected_overflow=0 exact_recv_per_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131045 expected_overflow=0 exact_recv_per_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131000 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131045 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=81920 expected_recv=127751 expected_overflow=0 exact_recv_per_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=81920 expected_recv=127801 expected_overflow=0 exact_recv_per_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=81920 expected_recv=127751 actual_valid_recv=128896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=81920 expected_recv=127801 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=128896 slack_slots=34944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=129152 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=81920 expected_recv=132028 expected_overflow=0 exact_recv_per_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=81920 expected_recv=132028 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=134400 slack_slots=29440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=81920 expected_recv=132378 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131461 expected_overflow=0 exact_recv_per_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131084 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131184 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=81920 expected_recv=132378 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131461 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131084 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131184 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=134144 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=81920 expected_recv=132388 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=81920 expected_recv=130775 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=81920 expected_recv=132388 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=81920 expected_recv=130775 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131058 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=81920 expected_recv=127325 expected_overflow=0 exact_recv_per_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=81920 expected_recv=130640 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131058 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=81920 expected_recv=127325 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=81920 expected_recv=130640 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=128512 slack_slots=35328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=81920 expected_recv=126316 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=81920 expected_recv=126316 actual_valid_recv=127488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=127488 slack_slots=36352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[54643, 63444, 57487, 89558, 66684, 72776, 49415, 70281] avg=65536.0000 max=89558 max_rank=3 min=49415 min_rank=6 imbalance_ratio=1.366547 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=81920 expected_recv=129703 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[54643, 63444, 57487, 89558, 66684, 72776, 49415, 70281] avg=65536.0000 max=89558 max_rank=3 min=49415 min_rank=6 imbalance_ratio=1.366547 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=81920 expected_recv=129703 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[54643, 63444, 57487, 89558, 66684, 72776, 49415, 70281] avg=65536.0000 max=89558 max_rank=3 min=49415 min_rank=6 imbalance_ratio=1.366547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[54643, 63444, 57487, 89558, 66684, 72776, 49415, 70281] avg=65536.0000 max=89558 max_rank=3 min=49415 min_rank=6 imbalance_ratio=1.366547 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=63936 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23365 max_load_reduction_pct=0.260892 moved_to_replicas=66582 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[54643, 63444, 57487, 89558, 66684, 72776, 49415, 70281] avg=65536.0000 max=89558 max_rank=3 min=49415 min_rank=6 imbalance_ratio=1.366547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[54643, 63444, 57487, 89558, 66684, 72776, 49415, 70281] avg=65536.0000 max=89558 max_rank=3 min=49415 min_rank=6 imbalance_ratio=1.366547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=63936 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23365 max_load_reduction_pct=0.260892 moved_to_replicas=66582 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=63936 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23365 max_load_reduction_pct=0.260892 moved_to_replicas=66582 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=63936 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23365 max_load_reduction_pct=0.260892 moved_to_replicas=66582 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=63936 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23365 max_load_reduction_pct=0.260892 moved_to_replicas=66582 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[65506, 79608, 30300, 80218, 32796, 63990, 65180, 106690] avg=65536.0000 max=106690 max_rank=7 min=30300 min_rank=2 imbalance_ratio=1.627960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=63936 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23365 max_load_reduction_pct=0.260892 moved_to_replicas=66582 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[54643, 63444, 57487, 89558, 66684, 72776, 49415, 70281] avg=65536.0000 max=89558 max_rank=3 min=49415 min_rank=6 imbalance_ratio=1.366547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[54643, 63444, 57487, 89558, 66684, 72776, 49415, 70281] avg=65536.0000 max=89558 max_rank=3 min=49415 min_rank=6 imbalance_ratio=1.366547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[65506, 79608, 30300, 80218, 32796, 63990, 65180, 106690] avg=65536.0000 max=106690 max_rank=7 min=30300 min_rank=2 imbalance_ratio=1.627960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[65506, 79608, 30300, 80218, 32796, 63990, 65180, 106690] avg=65536.0000 max=106690 max_rank=7 min=30300 min_rank=2 imbalance_ratio=1.627960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[65506, 79608, 30300, 80218, 32796, 63990, 65180, 106690] avg=65536.0000 max=106690 max_rank=7 min=30300 min_rank=2 imbalance_ratio=1.627960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] avg=65536.0000 max=66193 max_rank=1 min=63990 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=40497 max_load_reduction_pct=0.379576 moved_to_replicas=137428 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=63936 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23365 max_load_reduction_pct=0.260892 moved_to_replicas=66582 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[65192, 63936, 64199, 66192, 66192, 66193, 66192, 66192] avg=65536.0000 max=66193 max_rank=5 min=63936 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=23365 max_load_reduction_pct=0.260892 moved_to_replicas=66582 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[65506, 79608, 30300, 80218, 32796, 63990, 65180, 106690] avg=65536.0000 max=106690 max_rank=7 min=30300 min_rank=2 imbalance_ratio=1.627960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[65506, 79608, 30300, 80218, 32796, 63990, 65180, 106690] avg=65536.0000 max=106690 max_rank=7 min=30300 min_rank=2 imbalance_ratio=1.627960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] avg=65536.0000 max=66193 max_rank=1 min=63990 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=40497 max_load_reduction_pct=0.379576 moved_to_replicas=137428 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] avg=65536.0000 max=66193 max_rank=1 min=63990 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=40497 max_load_reduction_pct=0.379576 moved_to_replicas=137428 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] avg=65536.0000 max=66193 max_rank=1 min=63990 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=40497 max_load_reduction_pct=0.379576 moved_to_replicas=137428 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32504, 98411, 65574, 98381, 98572, 63124, 65558, 2164] avg=65536.0000 max=98572 max_rank=4 min=2164 min_rank=7 imbalance_ratio=1.504089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] avg=65536.0000 max=66193 max_rank=1 min=63990 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=40497 max_load_reduction_pct=0.379576 moved_to_replicas=137428 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] avg=65536.0000 max=66193 max_rank=1 min=63990 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=40497 max_load_reduction_pct=0.379576 moved_to_replicas=137428 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[65506, 79608, 30300, 80218, 32796, 63990, 65180, 106690] avg=65536.0000 max=106690 max_rank=7 min=30300 min_rank=2 imbalance_ratio=1.627960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[65506, 79608, 30300, 80218, 32796, 63990, 65180, 106690] avg=65536.0000 max=106690 max_rank=7 min=30300 min_rank=2 imbalance_ratio=1.627960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32504, 98411, 65574, 98381, 98572, 63124, 65558, 2164] avg=65536.0000 max=98572 max_rank=4 min=2164 min_rank=7 imbalance_ratio=1.504089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] avg=65536.0000 max=66192 max_rank=1 min=63665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32380 max_load_reduction_pct=0.328491 moved_to_replicas=193549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32504, 98411, 65574, 98381, 98572, 63124, 65558, 2164] avg=65536.0000 max=98572 max_rank=4 min=2164 min_rank=7 imbalance_ratio=1.504089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32504, 98411, 65574, 98381, 98572, 63124, 65558, 2164] avg=65536.0000 max=98572 max_rank=4 min=2164 min_rank=7 imbalance_ratio=1.504089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] avg=65536.0000 max=66193 max_rank=1 min=63990 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=40497 max_load_reduction_pct=0.379576 moved_to_replicas=137428 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32504, 98411, 65574, 98381, 98572, 63124, 65558, 2164] avg=65536.0000 max=98572 max_rank=4 min=2164 min_rank=7 imbalance_ratio=1.504089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[65506, 66193, 64844, 66192, 66192, 63990, 65180, 66191] avg=65536.0000 max=66193 max_rank=1 min=63990 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=40497 max_load_reduction_pct=0.379576 moved_to_replicas=137428 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32504, 98411, 65574, 98381, 98572, 63124, 65558, 2164] avg=65536.0000 max=98572 max_rank=4 min=2164 min_rank=7 imbalance_ratio=1.504089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] avg=65536.0000 max=66192 max_rank=1 min=63665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32380 max_load_reduction_pct=0.328491 moved_to_replicas=193549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] avg=65536.0000 max=66192 max_rank=1 min=63665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32380 max_load_reduction_pct=0.328491 moved_to_replicas=193549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] avg=65536.0000 max=66192 max_rank=1 min=63665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32380 max_load_reduction_pct=0.328491 moved_to_replicas=193549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65464, 130778, 87372, 32711, 64287, 32967, 47818, 62891] avg=65536.0000 max=130778 max_rank=1 min=32711 min_rank=3 imbalance_ratio=1.995514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] avg=65536.0000 max=66192 max_rank=1 min=63665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32380 max_load_reduction_pct=0.328491 moved_to_replicas=193549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] avg=65536.0000 max=66192 max_rank=1 min=63665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32380 max_load_reduction_pct=0.328491 moved_to_replicas=193549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32504, 98411, 65574, 98381, 98572, 63124, 65558, 2164] avg=65536.0000 max=98572 max_rank=4 min=2164 min_rank=7 imbalance_ratio=1.504089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32504, 98411, 65574, 98381, 98572, 63124, 65558, 2164] avg=65536.0000 max=98572 max_rank=4 min=2164 min_rank=7 imbalance_ratio=1.504089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] avg=65536.0000 max=66194 max_rank=2 min=64287 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64584 max_load_reduction_pct=0.493845 moved_to_replicas=171635 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65464, 130778, 87372, 32711, 64287, 32967, 47818, 62891] avg=65536.0000 max=130778 max_rank=1 min=32711 min_rank=3 imbalance_ratio=1.995514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65464, 130778, 87372, 32711, 64287, 32967, 47818, 62891] avg=65536.0000 max=130778 max_rank=1 min=32711 min_rank=3 imbalance_ratio=1.995514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65464, 130778, 87372, 32711, 64287, 32967, 47818, 62891] avg=65536.0000 max=130778 max_rank=1 min=32711 min_rank=3 imbalance_ratio=1.995514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] avg=65536.0000 max=66192 max_rank=1 min=63665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32380 max_load_reduction_pct=0.328491 moved_to_replicas=193549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65464, 130778, 87372, 32711, 64287, 32967, 47818, 62891] avg=65536.0000 max=130778 max_rank=1 min=32711 min_rank=3 imbalance_ratio=1.995514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65464, 130778, 87372, 32711, 64287, 32967, 47818, 62891] avg=65536.0000 max=130778 max_rank=1 min=32711 min_rank=3 imbalance_ratio=1.995514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[64723, 66192, 65574, 66192, 66192, 63665, 65558, 66192] avg=65536.0000 max=66192 max_rank=1 min=63665 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32380 max_load_reduction_pct=0.328491 moved_to_replicas=193549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] avg=65536.0000 max=66194 max_rank=2 min=64287 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64584 max_load_reduction_pct=0.493845 moved_to_replicas=171635 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] avg=65536.0000 max=66194 max_rank=2 min=64287 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64584 max_load_reduction_pct=0.493845 moved_to_replicas=171635 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] avg=65536.0000 max=66194 max_rank=2 min=64287 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64584 max_load_reduction_pct=0.493845 moved_to_replicas=171635 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32993, 98146, 845, 65544, 65802, 98277, 64527, 98154] avg=65536.0000 max=98277 max_rank=5 min=845 min_rank=2 imbalance_ratio=1.499588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] avg=65536.0000 max=66194 max_rank=2 min=64287 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64584 max_load_reduction_pct=0.493845 moved_to_replicas=171635 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] avg=65536.0000 max=66194 max_rank=2 min=64287 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64584 max_load_reduction_pct=0.493845 moved_to_replicas=171635 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65464, 130778, 87372, 32711, 64287, 32967, 47818, 62891] avg=65536.0000 max=130778 max_rank=1 min=32711 min_rank=3 imbalance_ratio=1.995514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65464, 130778, 87372, 32711, 64287, 32967, 47818, 62891] avg=65536.0000 max=130778 max_rank=1 min=32711 min_rank=3 imbalance_ratio=1.995514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] avg=65536.0000 max=66192 max_rank=1 min=64527 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32085 max_load_reduction_pct=0.326475 moved_to_replicas=191831 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32993, 98146, 845, 65544, 65802, 98277, 64527, 98154] avg=65536.0000 max=98277 max_rank=5 min=845 min_rank=2 imbalance_ratio=1.499588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32993, 98146, 845, 65544, 65802, 98277, 64527, 98154] avg=65536.0000 max=98277 max_rank=5 min=845 min_rank=2 imbalance_ratio=1.499588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32993, 98146, 845, 65544, 65802, 98277, 64527, 98154] avg=65536.0000 max=98277 max_rank=5 min=845 min_rank=2 imbalance_ratio=1.499588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] avg=65536.0000 max=66194 max_rank=2 min=64287 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64584 max_load_reduction_pct=0.493845 moved_to_replicas=171635 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32993, 98146, 845, 65544, 65802, 98277, 64527, 98154] avg=65536.0000 max=98277 max_rank=5 min=845 min_rank=2 imbalance_ratio=1.499588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32993, 98146, 845, 65544, 65802, 98277, 64527, 98154] avg=65536.0000 max=98277 max_rank=5 min=845 min_rank=2 imbalance_ratio=1.499588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65464, 66192, 66194, 65464, 64287, 64800, 66190, 65697] avg=65536.0000 max=66194 max_rank=2 min=64287 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=64584 max_load_reduction_pct=0.493845 moved_to_replicas=171635 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] avg=65536.0000 max=66192 max_rank=1 min=64527 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32085 max_load_reduction_pct=0.326475 moved_to_replicas=191831 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] avg=65536.0000 max=66192 max_rank=1 min=64527 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32085 max_load_reduction_pct=0.326475 moved_to_replicas=191831 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] avg=65536.0000 max=66192 max_rank=1 min=64527 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32085 max_load_reduction_pct=0.326475 moved_to_replicas=191831 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[33476, 33261, 64862, 98282, 131030, 98324, 35072, 29981] avg=65536.0000 max=131030 max_rank=4 min=29981 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] avg=65536.0000 max=66192 max_rank=1 min=64527 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32085 max_load_reduction_pct=0.326475 moved_to_replicas=191831 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] avg=65536.0000 max=66192 max_rank=1 min=64527 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32085 max_load_reduction_pct=0.326475 moved_to_replicas=191831 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32993, 98146, 845, 65544, 65802, 98277, 64527, 98154] avg=65536.0000 max=98277 max_rank=5 min=845 min_rank=2 imbalance_ratio=1.499588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32993, 98146, 845, 65544, 65802, 98277, 64527, 98154] avg=65536.0000 max=98277 max_rank=5 min=845 min_rank=2 imbalance_ratio=1.499588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] avg=65536.0000 max=66192 max_rank=3 min=63711 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64838 max_load_reduction_pct=0.494833 moved_to_replicas=258101 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[33476, 33261, 64862, 98282, 131030, 98324, 35072, 29981] avg=65536.0000 max=131030 max_rank=4 min=29981 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[33476, 33261, 64862, 98282, 131030, 98324, 35072, 29981] avg=65536.0000 max=131030 max_rank=4 min=29981 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[33476, 33261, 64862, 98282, 131030, 98324, 35072, 29981] avg=65536.0000 max=131030 max_rank=4 min=29981 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] avg=65536.0000 max=66192 max_rank=1 min=64527 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32085 max_load_reduction_pct=0.326475 moved_to_replicas=191831 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[33476, 33261, 64862, 98282, 131030, 98324, 35072, 29981] avg=65536.0000 max=131030 max_rank=4 min=29981 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[33476, 33261, 64862, 98282, 131030, 98324, 35072, 29981] avg=65536.0000 max=131030 max_rank=4 min=29981 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[64947, 66192, 64892, 65544, 65802, 66192, 64527, 66192] avg=65536.0000 max=66192 max_rank=1 min=64527 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32085 max_load_reduction_pct=0.326475 moved_to_replicas=191831 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] avg=65536.0000 max=66192 max_rank=3 min=63711 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64838 max_load_reduction_pct=0.494833 moved_to_replicas=258101 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] avg=65536.0000 max=66192 max_rank=3 min=63711 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64838 max_load_reduction_pct=0.494833 moved_to_replicas=258101 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] avg=65536.0000 max=66192 max_rank=3 min=63711 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64838 max_load_reduction_pct=0.494833 moved_to_replicas=258101 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32787, 95779, 98089, 32790, 18420, 92513, 120853, 33057] avg=65536.0000 max=120853 max_rank=6 min=18420 min_rank=4 imbalance_ratio=1.844070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] avg=65536.0000 max=66192 max_rank=3 min=63711 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64838 max_load_reduction_pct=0.494833 moved_to_replicas=258101 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] avg=65536.0000 max=66192 max_rank=3 min=63711 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64838 max_load_reduction_pct=0.494833 moved_to_replicas=258101 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[33476, 33261, 64862, 98282, 131030, 98324, 35072, 29981] avg=65536.0000 max=131030 max_rank=4 min=29981 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] avg=65536.0000 max=66193 max_rank=5 min=62644 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=54660 max_load_reduction_pct=0.452285 moved_to_replicas=284498 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[33476, 33261, 64862, 98282, 131030, 98324, 35072, 29981] avg=65536.0000 max=131030 max_rank=4 min=29981 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32787, 95779, 98089, 32790, 18420, 92513, 120853, 33057] avg=65536.0000 max=120853 max_rank=6 min=18420 min_rank=4 imbalance_ratio=1.844070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32787, 95779, 98089, 32790, 18420, 92513, 120853, 33057] avg=65536.0000 max=120853 max_rank=6 min=18420 min_rank=4 imbalance_ratio=1.844070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32787, 95779, 98089, 32790, 18420, 92513, 120853, 33057] avg=65536.0000 max=120853 max_rank=6 min=18420 min_rank=4 imbalance_ratio=1.844070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] avg=65536.0000 max=66192 max_rank=3 min=63711 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64838 max_load_reduction_pct=0.494833 moved_to_replicas=258101 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32787, 95779, 98089, 32790, 18420, 92513, 120853, 33057] avg=65536.0000 max=120853 max_rank=6 min=18420 min_rank=4 imbalance_ratio=1.844070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65608, 65339, 64862, 66192, 66192, 66192, 66192, 63711] avg=65536.0000 max=66192 max_rank=3 min=63711 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64838 max_load_reduction_pct=0.494833 moved_to_replicas=258101 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32787, 95779, 98089, 32790, 18420, 92513, 120853, 33057] avg=65536.0000 max=120853 max_rank=6 min=18420 min_rank=4 imbalance_ratio=1.844070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] avg=65536.0000 max=66193 max_rank=5 min=62644 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=54660 max_load_reduction_pct=0.452285 moved_to_replicas=284498 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] avg=65536.0000 max=66193 max_rank=5 min=62644 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=54660 max_load_reduction_pct=0.452285 moved_to_replicas=284498 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[66341, 53660, 124771, 148381, 32722, 31, 32782, 65600] avg=65536.0000 max=148381 max_rank=3 min=31 min_rank=5 imbalance_ratio=2.264114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] avg=65536.0000 max=66193 max_rank=5 min=62644 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=54660 max_load_reduction_pct=0.452285 moved_to_replicas=284498 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] avg=65536.0000 max=66193 max_rank=5 min=62644 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=54660 max_load_reduction_pct=0.452285 moved_to_replicas=284498 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32787, 95779, 98089, 32790, 18420, 92513, 120853, 33057] avg=65536.0000 max=120853 max_rank=6 min=18420 min_rank=4 imbalance_ratio=1.844070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] avg=65536.0000 max=66193 max_rank=5 min=62644 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=54660 max_load_reduction_pct=0.452285 moved_to_replicas=284498 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] avg=65536.0000 max=66193 max_rank=2 min=62903 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=82188 max_load_reduction_pct=0.553898 moved_to_replicas=280972 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32787, 95779, 98089, 32790, 18420, 92513, 120853, 33057] avg=65536.0000 max=120853 max_rank=6 min=18420 min_rank=4 imbalance_ratio=1.844070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[66341, 53660, 124771, 148381, 32722, 31, 32782, 65600] avg=65536.0000 max=148381 max_rank=3 min=31 min_rank=5 imbalance_ratio=2.264114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] avg=65536.0000 max=66193 max_rank=5 min=62644 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=54660 max_load_reduction_pct=0.452285 moved_to_replicas=284498 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[66341, 53660, 124771, 148381, 32722, 31, 32782, 65600] avg=65536.0000 max=148381 max_rank=3 min=31 min_rank=5 imbalance_ratio=2.264114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[66341, 53660, 124771, 148381, 32722, 31, 32782, 65600] avg=65536.0000 max=148381 max_rank=3 min=31 min_rank=5 imbalance_ratio=2.264114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[66341, 53660, 124771, 148381, 32722, 31, 32782, 65600] avg=65536.0000 max=148381 max_rank=3 min=31 min_rank=5 imbalance_ratio=2.264114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[66341, 53660, 124771, 148381, 32722, 31, 32782, 65600] avg=65536.0000 max=148381 max_rank=3 min=31 min_rank=5 imbalance_ratio=2.264114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65999, 66192, 66192, 64687, 66190, 66193, 66191, 62644] avg=65536.0000 max=66193 max_rank=5 min=62644 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=54660 max_load_reduction_pct=0.452285 moved_to_replicas=284498 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] avg=65536.0000 max=66193 max_rank=2 min=62903 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=82188 max_load_reduction_pct=0.553898 moved_to_replicas=280972 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[32788, 64420, 97, 66408, 67728, 64259, 130928, 97660] avg=65536.0000 max=130928 max_rank=6 min=97 min_rank=2 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] avg=65536.0000 max=66193 max_rank=2 min=62903 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=82188 max_load_reduction_pct=0.553898 moved_to_replicas=280972 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] avg=65536.0000 max=66193 max_rank=2 min=62903 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=82188 max_load_reduction_pct=0.553898 moved_to_replicas=280972 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] avg=65536.0000 max=66193 max_rank=2 min=62903 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=82188 max_load_reduction_pct=0.553898 moved_to_replicas=280972 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[66341, 53660, 124771, 148381, 32722, 31, 32782, 65600] avg=65536.0000 max=148381 max_rank=3 min=31 min_rank=5 imbalance_ratio=2.264114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] avg=65536.0000 max=66193 max_rank=2 min=62903 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=82188 max_load_reduction_pct=0.553898 moved_to_replicas=280972 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64420 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64736 max_load_reduction_pct=0.494440 moved_to_replicas=196222 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[32788, 64420, 97, 66408, 67728, 64259, 130928, 97660] avg=65536.0000 max=130928 max_rank=6 min=97 min_rank=2 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[66341, 53660, 124771, 148381, 32722, 31, 32782, 65600] avg=65536.0000 max=148381 max_rank=3 min=31 min_rank=5 imbalance_ratio=2.264114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] avg=65536.0000 max=66193 max_rank=2 min=62903 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=82188 max_load_reduction_pct=0.553898 moved_to_replicas=280972 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[32788, 64420, 97, 66408, 67728, 64259, 130928, 97660] avg=65536.0000 max=130928 max_rank=6 min=97 min_rank=2 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[32788, 64420, 97, 66408, 67728, 64259, 130928, 97660] avg=65536.0000 max=130928 max_rank=6 min=97 min_rank=2 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[32788, 64420, 97, 66408, 67728, 64259, 130928, 97660] avg=65536.0000 max=130928 max_rank=6 min=97 min_rank=2 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[32788, 64420, 97, 66408, 67728, 64259, 130928, 97660] avg=65536.0000 max=130928 max_rank=6 min=97 min_rank=2 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65516, 32872, 32762, 82811, 80683, 32958, 98383] avg=65536.0000 max=98383 max_rank=7 min=32762 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64420 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64736 max_load_reduction_pct=0.494440 moved_to_replicas=196222 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[66192, 62903, 66193, 66190, 65467, 65550, 66193, 65600] avg=65536.0000 max=66193 max_rank=2 min=62903 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=82188 max_load_reduction_pct=0.553898 moved_to_replicas=280972 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64420 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64736 max_load_reduction_pct=0.494440 moved_to_replicas=196222 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64420 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64736 max_load_reduction_pct=0.494440 moved_to_replicas=196222 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64420 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64736 max_load_reduction_pct=0.494440 moved_to_replicas=196222 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[32788, 64420, 97, 66408, 67728, 64259, 130928, 97660] avg=65536.0000 max=130928 max_rank=6 min=97 min_rank=2 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64420 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64736 max_load_reduction_pct=0.494440 moved_to_replicas=196222 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] avg=65536.0000 max=66192 max_rank=0 min=64069 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190896 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65516, 32872, 32762, 82811, 80683, 32958, 98383] avg=65536.0000 max=98383 max_rank=7 min=32762 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[32788, 64420, 97, 66408, 67728, 64259, 130928, 97660] avg=65536.0000 max=130928 max_rank=6 min=97 min_rank=2 imbalance_ratio=1.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64420 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64736 max_load_reduction_pct=0.494440 moved_to_replicas=196222 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65516, 32872, 32762, 82811, 80683, 32958, 98383] avg=65536.0000 max=98383 max_rank=7 min=32762 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65516, 32872, 32762, 82811, 80683, 32958, 98383] avg=65536.0000 max=98383 max_rank=7 min=32762 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65516, 32872, 32762, 82811, 80683, 32958, 98383] avg=65536.0000 max=98383 max_rank=7 min=32762 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] avg=65536.0000 max=66192 max_rank=0 min=64069 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190896 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[98221, 645, 32887, 99742, 98231, 67167, 92151, 35244] avg=65536.0000 max=99742 max_rank=3 min=645 min_rank=1 imbalance_ratio=1.521942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65516, 32872, 32762, 82811, 80683, 32958, 98383] avg=65536.0000 max=98383 max_rank=7 min=32762 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] avg=65536.0000 max=66192 max_rank=0 min=64069 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190896 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[64767, 64420, 64538, 66192, 66192, 65795, 66192, 66192] avg=65536.0000 max=66192 max_rank=3 min=64420 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64736 max_load_reduction_pct=0.494440 moved_to_replicas=196222 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] avg=65536.0000 max=66192 max_rank=0 min=64069 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190896 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] avg=65536.0000 max=66192 max_rank=0 min=64069 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190896 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65516, 32872, 32762, 82811, 80683, 32958, 98383] avg=65536.0000 max=98383 max_rank=7 min=32762 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] avg=65536.0000 max=66192 max_rank=0 min=62178 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33550 max_load_reduction_pct=0.336368 moved_to_replicas=248627 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] avg=65536.0000 max=66192 max_rank=0 min=64069 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190896 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[98221, 645, 32887, 99742, 98231, 67167, 92151, 35244] avg=65536.0000 max=99742 max_rank=3 min=645 min_rank=1 imbalance_ratio=1.521942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] avg=65536.0000 max=66192 max_rank=0 min=64069 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190896 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[98221, 645, 32887, 99742, 98231, 67167, 92151, 35244] avg=65536.0000 max=99742 max_rank=3 min=645 min_rank=1 imbalance_ratio=1.521942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[98303, 65516, 32872, 32762, 82811, 80683, 32958, 98383] avg=65536.0000 max=98383 max_rank=7 min=32762 min_rank=3 imbalance_ratio=1.501205 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[98221, 645, 32887, 99742, 98231, 67167, 92151, 35244] avg=65536.0000 max=99742 max_rank=3 min=645 min_rank=1 imbalance_ratio=1.521942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[98221, 645, 32887, 99742, 98231, 67167, 92151, 35244] avg=65536.0000 max=99742 max_rank=3 min=645 min_rank=1 imbalance_ratio=1.521942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] avg=65536.0000 max=66192 max_rank=0 min=62178 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33550 max_load_reduction_pct=0.336368 moved_to_replicas=248627 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[65369, 34419, 98453, 130311, 27, 130584, 337, 64788] avg=65536.0000 max=130584 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.992554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[98221, 645, 32887, 99742, 98231, 67167, 92151, 35244] avg=65536.0000 max=99742 max_rank=3 min=645 min_rank=1 imbalance_ratio=1.521942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] avg=65536.0000 max=66192 max_rank=0 min=62178 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33550 max_load_reduction_pct=0.336368 moved_to_replicas=248627 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[66192, 65516, 64983, 64953, 66191, 66192, 64069, 66192] avg=65536.0000 max=66192 max_rank=0 min=64069 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32191 max_load_reduction_pct=0.327201 moved_to_replicas=190896 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] avg=65536.0000 max=66192 max_rank=0 min=62178 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33550 max_load_reduction_pct=0.336368 moved_to_replicas=248627 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] avg=65536.0000 max=66192 max_rank=0 min=62178 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33550 max_load_reduction_pct=0.336368 moved_to_replicas=248627 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[98221, 645, 32887, 99742, 98231, 67167, 92151, 35244] avg=65536.0000 max=99742 max_rank=3 min=645 min_rank=1 imbalance_ratio=1.521942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] avg=65536.0000 max=66192 max_rank=1 min=64635 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64392 max_load_reduction_pct=0.493108 moved_to_replicas=321801 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] avg=65536.0000 max=66192 max_rank=0 min=62178 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33550 max_load_reduction_pct=0.336368 moved_to_replicas=248627 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[65369, 34419, 98453, 130311, 27, 130584, 337, 64788] avg=65536.0000 max=130584 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.992554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] avg=65536.0000 max=66192 max_rank=0 min=62178 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33550 max_load_reduction_pct=0.336368 moved_to_replicas=248627 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[65369, 34419, 98453, 130311, 27, 130584, 337, 64788] avg=65536.0000 max=130584 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.992554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[98221, 645, 32887, 99742, 98231, 67167, 92151, 35244] avg=65536.0000 max=99742 max_rank=3 min=645 min_rank=1 imbalance_ratio=1.521942 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[65369, 34419, 98453, 130311, 27, 130584, 337, 64788] avg=65536.0000 max=130584 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.992554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[65369, 34419, 98453, 130311, 27, 130584, 337, 64788] avg=65536.0000 max=130584 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.992554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] avg=65536.0000 max=66192 max_rank=1 min=64635 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64392 max_load_reduction_pct=0.493108 moved_to_replicas=321801 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[32804, 17784, 51691, 98049, 65532, 126114, 66786, 65528] avg=65536.0000 max=126114 max_rank=5 min=17784 min_rank=1 imbalance_ratio=1.924347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] avg=65536.0000 max=66192 max_rank=1 min=64635 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64392 max_load_reduction_pct=0.493108 moved_to_replicas=321801 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[65369, 34419, 98453, 130311, 27, 130584, 337, 64788] avg=65536.0000 max=130584 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.992554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[66192, 65438, 65712, 66192, 66192, 66192, 66192, 62178] avg=65536.0000 max=66192 max_rank=0 min=62178 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33550 max_load_reduction_pct=0.336368 moved_to_replicas=248627 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] avg=65536.0000 max=66192 max_rank=1 min=64635 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64392 max_load_reduction_pct=0.493108 moved_to_replicas=321801 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] avg=65536.0000 max=66192 max_rank=1 min=64635 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64392 max_load_reduction_pct=0.493108 moved_to_replicas=321801 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[65369, 34419, 98453, 130311, 27, 130584, 337, 64788] avg=65536.0000 max=130584 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.992554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] avg=65536.0000 max=66193 max_rank=2 min=62269 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=59921 max_load_reduction_pct=0.475134 moved_to_replicas=186738 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] avg=65536.0000 max=66192 max_rank=1 min=64635 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64392 max_load_reduction_pct=0.493108 moved_to_replicas=321801 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[32804, 17784, 51691, 98049, 65532, 126114, 66786, 65528] avg=65536.0000 max=126114 max_rank=5 min=17784 min_rank=1 imbalance_ratio=1.924347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[32804, 17784, 51691, 98049, 65532, 126114, 66786, 65528] avg=65536.0000 max=126114 max_rank=5 min=17784 min_rank=1 imbalance_ratio=1.924347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] avg=65536.0000 max=66192 max_rank=1 min=64635 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64392 max_load_reduction_pct=0.493108 moved_to_replicas=321801 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[32804, 17784, 51691, 98049, 65532, 126114, 66786, 65528] avg=65536.0000 max=126114 max_rank=5 min=17784 min_rank=1 imbalance_ratio=1.924347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[32804, 17784, 51691, 98049, 65532, 126114, 66786, 65528] avg=65536.0000 max=126114 max_rank=5 min=17784 min_rank=1 imbalance_ratio=1.924347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[65369, 34419, 98453, 130311, 27, 130584, 337, 64788] avg=65536.0000 max=130584 max_rank=5 min=27 min_rank=4 imbalance_ratio=1.992554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] avg=65536.0000 max=66193 max_rank=2 min=62269 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=59921 max_load_reduction_pct=0.475134 moved_to_replicas=186738 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] avg=65536.0000 max=66193 max_rank=2 min=62269 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=59921 max_load_reduction_pct=0.475134 moved_to_replicas=186738 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[110447, 32866, 128257, 32871, 36870, 33000, 93601, 56376] avg=65536.0000 max=128257 max_rank=2 min=32866 min_rank=1 imbalance_ratio=1.957047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[32804, 17784, 51691, 98049, 65532, 126114, 66786, 65528] avg=65536.0000 max=126114 max_rank=5 min=17784 min_rank=1 imbalance_ratio=1.924347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] avg=65536.0000 max=66193 max_rank=2 min=62269 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=59921 max_load_reduction_pct=0.475134 moved_to_replicas=186738 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] avg=65536.0000 max=66193 max_rank=2 min=62269 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=59921 max_load_reduction_pct=0.475134 moved_to_replicas=186738 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[32804, 17784, 51691, 98049, 65532, 126114, 66786, 65528] avg=65536.0000 max=126114 max_rank=5 min=17784 min_rank=1 imbalance_ratio=1.924347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[65369, 66192, 66192, 66192, 64635, 66192, 64728, 64788] avg=65536.0000 max=66192 max_rank=1 min=64635 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64392 max_load_reduction_pct=0.493108 moved_to_replicas=321801 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] avg=65536.0000 max=66193 max_rank=4 min=62178 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62064 max_load_reduction_pct=0.483903 moved_to_replicas=266126 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] avg=65536.0000 max=66193 max_rank=2 min=62269 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=59921 max_load_reduction_pct=0.475134 moved_to_replicas=186738 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[110447, 32866, 128257, 32871, 36870, 33000, 93601, 56376] avg=65536.0000 max=128257 max_rank=2 min=32866 min_rank=1 imbalance_ratio=1.957047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[110447, 32866, 128257, 32871, 36870, 33000, 93601, 56376] avg=65536.0000 max=128257 max_rank=2 min=32866 min_rank=1 imbalance_ratio=1.957047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] avg=65536.0000 max=66193 max_rank=2 min=62269 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=59921 max_load_reduction_pct=0.475134 moved_to_replicas=186738 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[110447, 32866, 128257, 32871, 36870, 33000, 93601, 56376] avg=65536.0000 max=128257 max_rank=2 min=32866 min_rank=1 imbalance_ratio=1.957047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[110447, 32866, 128257, 32871, 36870, 33000, 93601, 56376] avg=65536.0000 max=128257 max_rank=2 min=32866 min_rank=1 imbalance_ratio=1.957047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[32804, 17784, 51691, 98049, 65532, 126114, 66786, 65528] avg=65536.0000 max=126114 max_rank=5 min=17784 min_rank=1 imbalance_ratio=1.924347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] avg=65536.0000 max=66193 max_rank=4 min=62178 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62064 max_load_reduction_pct=0.483903 moved_to_replicas=266126 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[97576, 32801, 65583, 32778, 108304, 56350, 65528, 65368] avg=65536.0000 max=108304 max_rank=4 min=32778 min_rank=3 imbalance_ratio=1.652588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] avg=65536.0000 max=66193 max_rank=4 min=62178 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62064 max_load_reduction_pct=0.483903 moved_to_replicas=266126 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[110447, 32866, 128257, 32871, 36870, 33000, 93601, 56376] avg=65536.0000 max=128257 max_rank=2 min=32866 min_rank=1 imbalance_ratio=1.957047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] avg=65536.0000 max=66193 max_rank=4 min=62178 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62064 max_load_reduction_pct=0.483903 moved_to_replicas=266126 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] avg=65536.0000 max=66193 max_rank=4 min=62178 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62064 max_load_reduction_pct=0.483903 moved_to_replicas=266126 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[110447, 32866, 128257, 32871, 36870, 33000, 93601, 56376] avg=65536.0000 max=128257 max_rank=2 min=32866 min_rank=1 imbalance_ratio=1.957047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[62269, 66191, 66193, 66192, 65532, 66191, 66192, 65528] avg=65536.0000 max=66193 max_rank=2 min=62269 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=59921 max_load_reduction_pct=0.475134 moved_to_replicas=186738 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] avg=65536.0000 max=66194 max_rank=4 min=63696 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=42110 max_load_reduction_pct=0.388813 moved_to_replicas=145723 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] avg=65536.0000 max=66193 max_rank=4 min=62178 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62064 max_load_reduction_pct=0.483903 moved_to_replicas=266126 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[97576, 32801, 65583, 32778, 108304, 56350, 65528, 65368] avg=65536.0000 max=108304 max_rank=4 min=32778 min_rank=3 imbalance_ratio=1.652588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] avg=65536.0000 max=66193 max_rank=4 min=62178 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62064 max_load_reduction_pct=0.483903 moved_to_replicas=266126 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[97576, 32801, 65583, 32778, 108304, 56350, 65528, 65368] avg=65536.0000 max=108304 max_rank=4 min=32778 min_rank=3 imbalance_ratio=1.652588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[97576, 32801, 65583, 32778, 108304, 56350, 65528, 65368] avg=65536.0000 max=108304 max_rank=4 min=32778 min_rank=3 imbalance_ratio=1.652588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[97576, 32801, 65583, 32778, 108304, 56350, 65528, 65368] avg=65536.0000 max=108304 max_rank=4 min=32778 min_rank=3 imbalance_ratio=1.652588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] avg=65536.0000 max=66194 max_rank=4 min=63696 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=42110 max_load_reduction_pct=0.388813 moved_to_replicas=145723 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[110447, 32866, 128257, 32871, 36870, 33000, 93601, 56376] avg=65536.0000 max=128257 max_rank=2 min=32866 min_rank=1 imbalance_ratio=1.957047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[40713, 67060, 98487, 65267, 56357, 709, 97479, 98216] avg=65536.0000 max=98487 max_rank=2 min=709 min_rank=5 imbalance_ratio=1.502792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] avg=65536.0000 max=66194 max_rank=4 min=63696 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=42110 max_load_reduction_pct=0.388813 moved_to_replicas=145723 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[97576, 32801, 65583, 32778, 108304, 56350, 65528, 65368] avg=65536.0000 max=108304 max_rank=4 min=32778 min_rank=3 imbalance_ratio=1.652588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] avg=65536.0000 max=66194 max_rank=4 min=63696 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=42110 max_load_reduction_pct=0.388813 moved_to_replicas=145723 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[97576, 32801, 65583, 32778, 108304, 56350, 65528, 65368] avg=65536.0000 max=108304 max_rank=4 min=32778 min_rank=3 imbalance_ratio=1.652588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] avg=65536.0000 max=66194 max_rank=4 min=63696 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=42110 max_load_reduction_pct=0.388813 moved_to_replicas=145723 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[66191, 65624, 66192, 62178, 66193, 65756, 66191, 65963] avg=65536.0000 max=66193 max_rank=4 min=62178 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=62064 max_load_reduction_pct=0.483903 moved_to_replicas=266126 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63035 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.327911 moved_to_replicas=193235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] avg=65536.0000 max=66194 max_rank=4 min=63696 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=42110 max_load_reduction_pct=0.388813 moved_to_replicas=145723 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[40713, 67060, 98487, 65267, 56357, 709, 97479, 98216] avg=65536.0000 max=98487 max_rank=2 min=709 min_rank=5 imbalance_ratio=1.502792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] avg=65536.0000 max=66194 max_rank=4 min=63696 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=42110 max_load_reduction_pct=0.388813 moved_to_replicas=145723 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[40713, 67060, 98487, 65267, 56357, 709, 97479, 98216] avg=65536.0000 max=98487 max_rank=2 min=709 min_rank=5 imbalance_ratio=1.502792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[40713, 67060, 98487, 65267, 56357, 709, 97479, 98216] avg=65536.0000 max=98487 max_rank=2 min=709 min_rank=5 imbalance_ratio=1.502792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[40713, 67060, 98487, 65267, 56357, 709, 97479, 98216] avg=65536.0000 max=98487 max_rank=2 min=709 min_rank=5 imbalance_ratio=1.502792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63035 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.327911 moved_to_replicas=193235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[97576, 32801, 65583, 32778, 108304, 56350, 65528, 65368] avg=65536.0000 max=108304 max_rank=4 min=32778 min_rank=3 imbalance_ratio=1.652588 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63035 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.327911 moved_to_replicas=193235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[40713, 67060, 98487, 65267, 56357, 709, 97479, 98216] avg=65536.0000 max=98487 max_rank=2 min=709 min_rank=5 imbalance_ratio=1.502792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[40713, 67060, 98487, 65267, 56357, 709, 97479, 98216] avg=65536.0000 max=98487 max_rank=2 min=709 min_rank=5 imbalance_ratio=1.502792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63035 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.327911 moved_to_replicas=193235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63035 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.327911 moved_to_replicas=193235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[66192, 66190, 65583, 65537, 66194, 63696, 65528, 65368] avg=65536.0000 max=66194 max_rank=4 min=63696 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=42110 max_load_reduction_pct=0.388813 moved_to_replicas=145723 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63035 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.327911 moved_to_replicas=193235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63035 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.327911 moved_to_replicas=193235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[40713, 67060, 98487, 65267, 56357, 709, 97479, 98216] avg=65536.0000 max=98487 max_rank=2 min=709 min_rank=5 imbalance_ratio=1.502792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[66190, 66192, 66192, 65267, 63035, 65028, 66192, 66192] avg=65536.0000 max=66192 max_rank=1 min=63035 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.327911 moved_to_replicas=193235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=2093997 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507904 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=2100641 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=500608 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=2093924 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508800 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=2102127 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=499968 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=2090859 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=511232 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=2103108 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=499072 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=2097412 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=503680 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=2095148 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507008 peak_demand=66193 required_capacity=1.010025 + [2026-08-30 05:14:13] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 529.0 | throughput per GPU (TFLOP/s/GPU): 630.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.113233E+00 | loss scale: 1.0 | grad norm: 5.425 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=81920 expected_recv=130643 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=81920 expected_recv=130562 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=81920 expected_recv=128667 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=81920 expected_recv=129168 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=81920 expected_recv=130643 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=81920 expected_recv=130562 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=81920 expected_recv=128667 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=81920 expected_recv=129168 actual_valid_recv=130304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=130048 slack_slots=33792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=130304 slack_slots=33536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=81920 expected_recv=130433 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=81920 expected_recv=130433 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131374 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=81920 expected_recv=129361 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=81920 expected_recv=128981 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131374 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131278 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=81920 expected_recv=129361 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=81920 expected_recv=128981 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131278 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=81920 expected_recv=125984 expected_overflow=0 exact_recv_per_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=81920 expected_recv=125984 actual_valid_recv=127616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=127616 slack_slots=36224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131100 expected_overflow=0 exact_recv_per_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=81920 expected_recv=129570 expected_overflow=0 exact_recv_per_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131100 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=81920 expected_recv=129570 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=81920 expected_recv=130351 expected_overflow=0 exact_recv_per_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=81920 expected_recv=123920 expected_overflow=0 exact_recv_per_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=81920 expected_recv=130351 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=81920 expected_recv=123920 actual_valid_recv=125184 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=125184 slack_slots=38656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132389 expected_overflow=0 exact_recv_per_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132389 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=81920 expected_recv=130485 expected_overflow=0 exact_recv_per_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=81920 expected_recv=130485 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=81920 expected_recv=132103 expected_overflow=0 exact_recv_per_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=81920 expected_recv=128928 expected_overflow=0 exact_recv_per_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131543 expected_overflow=0 exact_recv_per_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=81920 expected_recv=128928 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=81920 expected_recv=132103 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=81920 expected_recv=128367 expected_overflow=0 exact_recv_per_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131543 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=129792 slack_slots=34048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=81920 expected_recv=131059 expected_overflow=0 exact_recv_per_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=81920 expected_recv=128367 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=81920 expected_recv=131059 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=81920 expected_recv=126657 expected_overflow=0 exact_recv_per_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=81920 expected_recv=128940 expected_overflow=0 exact_recv_per_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=81920 expected_recv=126657 actual_valid_recv=127872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=81920 expected_recv=128940 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=127872 slack_slots=35968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=81920 expected_recv=126380 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=81920 expected_recv=126380 actual_valid_recv=128000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=128000 slack_slots=35840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=81920 expected_recv=131091 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=81920 expected_recv=131014 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130556 expected_overflow=0 exact_recv_per_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=81920 expected_recv=131014 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=81920 expected_recv=131091 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130556 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131334 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131334 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131146 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=81920 expected_recv=124176 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=81920 expected_recv=130991 expected_overflow=0 exact_recv_per_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131146 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=81920 expected_recv=124176 actual_valid_recv=125440 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=81920 expected_recv=130991 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=125440 slack_slots=38400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=81920 expected_recv=132380 expected_overflow=0 exact_recv_per_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=81920 expected_recv=132380 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=81920 expected_recv=132185 expected_overflow=0 exact_recv_per_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=81920 expected_recv=125440 expected_overflow=0 exact_recv_per_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131751 expected_overflow=0 exact_recv_per_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131060 expected_overflow=0 exact_recv_per_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131751 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=81920 expected_recv=132185 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=81920 expected_recv=125440 actual_valid_recv=127360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131060 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=127360 slack_slots=36480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131126 expected_overflow=0 exact_recv_per_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131126 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=81920 expected_recv=129757 expected_overflow=0 exact_recv_per_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131004 expected_overflow=0 exact_recv_per_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=81920 expected_recv=127153 expected_overflow=0 exact_recv_per_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=81920 expected_recv=129757 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131004 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=81920 expected_recv=127153 actual_valid_recv=128384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=128384 slack_slots=35456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=81920 expected_recv=127362 expected_overflow=0 exact_recv_per_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=81920 expected_recv=127362 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=129152 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=81920 expected_recv=129689 expected_overflow=0 exact_recv_per_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=81920 expected_recv=129606 expected_overflow=0 exact_recv_per_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=81920 expected_recv=130277 expected_overflow=0 exact_recv_per_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=81920 expected_recv=129689 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=81920 expected_recv=129606 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=81920 expected_recv=130277 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=81920 expected_recv=129670 expected_overflow=0 exact_recv_per_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=81920 expected_recv=129670 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=81920 expected_recv=130975 expected_overflow=0 exact_recv_per_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131157 expected_overflow=0 exact_recv_per_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=81920 expected_recv=129343 expected_overflow=0 exact_recv_per_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=81920 expected_recv=130975 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131157 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=81920 expected_recv=129343 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=81920 expected_recv=129273 expected_overflow=0 exact_recv_per_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=81920 expected_recv=129273 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=130048 slack_slots=33792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131049 expected_overflow=0 exact_recv_per_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=81920 expected_recv=126332 expected_overflow=0 exact_recv_per_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131049 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131053 expected_overflow=0 exact_recv_per_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=81920 expected_recv=130801 expected_overflow=0 exact_recv_per_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=81920 expected_recv=126332 actual_valid_recv=126976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131053 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=134144 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=81920 expected_recv=130801 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=126976 slack_slots=36864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=81920 expected_recv=128803 expected_overflow=0 exact_recv_per_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=81920 expected_recv=128803 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=81920 expected_recv=128382 expected_overflow=0 exact_recv_per_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=81920 expected_recv=131073 expected_overflow=0 exact_recv_per_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=81920 expected_recv=128382 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=81920 expected_recv=131073 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=81920 expected_recv=124511 expected_overflow=0 exact_recv_per_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=81920 expected_recv=124511 actual_valid_recv=125952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=81920 expected_recv=131070 expected_overflow=0 exact_recv_per_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=125952 slack_slots=37888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=81920 expected_recv=131070 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=81920 expected_recv=128984 expected_overflow=0 exact_recv_per_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=81920 expected_recv=130570 expected_overflow=0 exact_recv_per_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=81920 expected_recv=128984 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=81920 expected_recv=130570 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=81920 expected_recv=132288 expected_overflow=0 exact_recv_per_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=81920 expected_recv=132288 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=81920 expected_recv=130984 expected_overflow=0 exact_recv_per_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=81920 expected_recv=130984 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131241 expected_overflow=0 exact_recv_per_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=81920 expected_recv=129741 expected_overflow=0 exact_recv_per_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[62368, 71740, 58243, 87423, 66305, 64370, 49178, 64661] avg=65536.0000 max=87423 max_rank=3 min=49178 min_rank=6 imbalance_ratio=1.333969 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131241 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[62368, 71740, 58243, 87423, 66305, 64370, 49178, 64661] avg=65536.0000 max=87423 max_rank=3 min=49178 min_rank=6 imbalance_ratio=1.333969 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=81920 expected_recv=129741 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[62368, 71740, 58243, 87423, 66305, 64370, 49178, 64661] avg=65536.0000 max=87423 max_rank=3 min=49178 min_rank=6 imbalance_ratio=1.333969 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[62368, 71740, 58243, 87423, 66305, 64370, 49178, 64661] avg=65536.0000 max=87423 max_rank=3 min=49178 min_rank=6 imbalance_ratio=1.333969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] avg=65536.0000 max=66192 max_rank=0 min=64298 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21231 max_load_reduction_pct=0.242854 moved_to_replicas=55214 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] avg=65536.0000 max=66192 max_rank=0 min=64298 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21231 max_load_reduction_pct=0.242854 moved_to_replicas=55214 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] avg=65536.0000 max=66192 max_rank=0 min=64298 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21231 max_load_reduction_pct=0.242854 moved_to_replicas=55214 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[62368, 71740, 58243, 87423, 66305, 64370, 49178, 64661] avg=65536.0000 max=87423 max_rank=3 min=49178 min_rank=6 imbalance_ratio=1.333969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[62368, 71740, 58243, 87423, 66305, 64370, 49178, 64661] avg=65536.0000 max=87423 max_rank=3 min=49178 min_rank=6 imbalance_ratio=1.333969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] avg=65536.0000 max=66192 max_rank=0 min=64298 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21231 max_load_reduction_pct=0.242854 moved_to_replicas=55214 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[62368, 71740, 58243, 87423, 66305, 64370, 49178, 64661] avg=65536.0000 max=87423 max_rank=3 min=49178 min_rank=6 imbalance_ratio=1.333969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[62368, 71740, 58243, 87423, 66305, 64370, 49178, 64661] avg=65536.0000 max=87423 max_rank=3 min=49178 min_rank=6 imbalance_ratio=1.333969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[61671, 115833, 45335, 60314, 32622, 90366, 55358, 62789] avg=65536.0000 max=115833 max_rank=1 min=32622 min_rank=4 imbalance_ratio=1.767471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] avg=65536.0000 max=66192 max_rank=0 min=64298 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21231 max_load_reduction_pct=0.242854 moved_to_replicas=55214 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[61671, 115833, 45335, 60314, 32622, 90366, 55358, 62789] avg=65536.0000 max=115833 max_rank=1 min=32622 min_rank=4 imbalance_ratio=1.767471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] avg=65536.0000 max=66192 max_rank=0 min=64298 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21231 max_load_reduction_pct=0.242854 moved_to_replicas=55214 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[61671, 115833, 45335, 60314, 32622, 90366, 55358, 62789] avg=65536.0000 max=115833 max_rank=1 min=32622 min_rank=4 imbalance_ratio=1.767471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] avg=65536.0000 max=66192 max_rank=0 min=64298 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21231 max_load_reduction_pct=0.242854 moved_to_replicas=55214 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[66192, 66192, 64298, 66192, 66192, 64370, 66191, 64661] avg=65536.0000 max=66192 max_rank=0 min=64298 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21231 max_load_reduction_pct=0.242854 moved_to_replicas=55214 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] avg=65536.0000 max=66193 max_rank=3 min=62789 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=49640 max_load_reduction_pct=0.428548 moved_to_replicas=152675 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[61671, 115833, 45335, 60314, 32622, 90366, 55358, 62789] avg=65536.0000 max=115833 max_rank=1 min=32622 min_rank=4 imbalance_ratio=1.767471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] avg=65536.0000 max=66193 max_rank=3 min=62789 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=49640 max_load_reduction_pct=0.428548 moved_to_replicas=152675 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] avg=65536.0000 max=66193 max_rank=3 min=62789 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=49640 max_load_reduction_pct=0.428548 moved_to_replicas=152675 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[61671, 115833, 45335, 60314, 32622, 90366, 55358, 62789] avg=65536.0000 max=115833 max_rank=1 min=32622 min_rank=4 imbalance_ratio=1.767471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[61671, 115833, 45335, 60314, 32622, 90366, 55358, 62789] avg=65536.0000 max=115833 max_rank=1 min=32622 min_rank=4 imbalance_ratio=1.767471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] avg=65536.0000 max=66193 max_rank=3 min=62789 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=49640 max_load_reduction_pct=0.428548 moved_to_replicas=152675 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[61671, 115833, 45335, 60314, 32622, 90366, 55358, 62789] avg=65536.0000 max=115833 max_rank=1 min=32622 min_rank=4 imbalance_ratio=1.767471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[61671, 115833, 45335, 60314, 32622, 90366, 55358, 62789] avg=65536.0000 max=115833 max_rank=1 min=32622 min_rank=4 imbalance_ratio=1.767471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[533, 111864, 98206, 97465, 121751, 28867, 65557, 45] avg=65536.0000 max=121751 max_rank=4 min=45 min_rank=7 imbalance_ratio=1.857773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] avg=65536.0000 max=66193 max_rank=3 min=62789 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=49640 max_load_reduction_pct=0.428548 moved_to_replicas=152675 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[533, 111864, 98206, 97465, 121751, 28867, 65557, 45] avg=65536.0000 max=121751 max_rank=4 min=45 min_rank=7 imbalance_ratio=1.857773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] avg=65536.0000 max=66193 max_rank=3 min=62789 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=49640 max_load_reduction_pct=0.428548 moved_to_replicas=152675 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[533, 111864, 98206, 97465, 121751, 28867, 65557, 45] avg=65536.0000 max=121751 max_rank=4 min=45 min_rank=7 imbalance_ratio=1.857773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] avg=65536.0000 max=66193 max_rank=3 min=62789 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=49640 max_load_reduction_pct=0.428548 moved_to_replicas=152675 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[66192, 66191, 65182, 66193, 65358, 66191, 66192, 62789] avg=65536.0000 max=66193 max_rank=3 min=62789 min_rank=7 imbalance_ratio=1.010025 max_load_reduction=49640 max_load_reduction_pct=0.428548 moved_to_replicas=152675 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] avg=65536.0000 max=66193 max_rank=5 min=62966 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=55558 max_load_reduction_pct=0.456325 moved_to_replicas=330476 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[533, 111864, 98206, 97465, 121751, 28867, 65557, 45] avg=65536.0000 max=121751 max_rank=4 min=45 min_rank=7 imbalance_ratio=1.857773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] avg=65536.0000 max=66193 max_rank=5 min=62966 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=55558 max_load_reduction_pct=0.456325 moved_to_replicas=330476 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[533, 111864, 98206, 97465, 121751, 28867, 65557, 45] avg=65536.0000 max=121751 max_rank=4 min=45 min_rank=7 imbalance_ratio=1.857773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] avg=65536.0000 max=66193 max_rank=5 min=62966 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=55558 max_load_reduction_pct=0.456325 moved_to_replicas=330476 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[533, 111864, 98206, 97465, 121751, 28867, 65557, 45] avg=65536.0000 max=121751 max_rank=4 min=45 min_rank=7 imbalance_ratio=1.857773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] avg=65536.0000 max=66193 max_rank=5 min=62966 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=55558 max_load_reduction_pct=0.456325 moved_to_replicas=330476 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[533, 111864, 98206, 97465, 121751, 28867, 65557, 45] avg=65536.0000 max=121751 max_rank=4 min=45 min_rank=7 imbalance_ratio=1.857773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[58003, 108262, 102850, 472, 95992, 65331, 53944, 39434] avg=65536.0000 max=108262 max_rank=1 min=472 min_rank=3 imbalance_ratio=1.651947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[533, 111864, 98206, 97465, 121751, 28867, 65557, 45] avg=65536.0000 max=121751 max_rank=4 min=45 min_rank=7 imbalance_ratio=1.857773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] avg=65536.0000 max=66193 max_rank=5 min=62966 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=55558 max_load_reduction_pct=0.456325 moved_to_replicas=330476 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] avg=65536.0000 max=66193 max_rank=5 min=62966 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=55558 max_load_reduction_pct=0.456325 moved_to_replicas=330476 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[58003, 108262, 102850, 472, 95992, 65331, 53944, 39434] avg=65536.0000 max=108262 max_rank=1 min=472 min_rank=3 imbalance_ratio=1.651947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] avg=65536.0000 max=66193 max_rank=5 min=62966 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=55558 max_load_reduction_pct=0.456325 moved_to_replicas=330476 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[58003, 108262, 102850, 472, 95992, 65331, 53944, 39434] avg=65536.0000 max=108262 max_rank=1 min=472 min_rank=3 imbalance_ratio=1.651947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] avg=65536.0000 max=66194 max_rank=3 min=61805 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42068 max_load_reduction_pct=0.388576 moved_to_replicas=215092 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[62966, 66192, 66192, 66192, 66192, 66193, 65557, 64804] avg=65536.0000 max=66193 max_rank=5 min=62966 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=55558 max_load_reduction_pct=0.456325 moved_to_replicas=330476 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[58003, 108262, 102850, 472, 95992, 65331, 53944, 39434] avg=65536.0000 max=108262 max_rank=1 min=472 min_rank=3 imbalance_ratio=1.651947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] avg=65536.0000 max=66194 max_rank=3 min=61805 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42068 max_load_reduction_pct=0.388576 moved_to_replicas=215092 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[58003, 108262, 102850, 472, 95992, 65331, 53944, 39434] avg=65536.0000 max=108262 max_rank=1 min=472 min_rank=3 imbalance_ratio=1.651947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[58003, 108262, 102850, 472, 95992, 65331, 53944, 39434] avg=65536.0000 max=108262 max_rank=1 min=472 min_rank=3 imbalance_ratio=1.651947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] avg=65536.0000 max=66194 max_rank=3 min=61805 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42068 max_load_reduction_pct=0.388576 moved_to_replicas=215092 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[58003, 108262, 102850, 472, 95992, 65331, 53944, 39434] avg=65536.0000 max=108262 max_rank=1 min=472 min_rank=3 imbalance_ratio=1.651947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] avg=65536.0000 max=66194 max_rank=3 min=61805 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42068 max_load_reduction_pct=0.388576 moved_to_replicas=215092 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[65349, 80331, 19066, 65857, 64211, 98896, 32879, 97699] avg=65536.0000 max=98896 max_rank=5 min=19066 min_rank=2 imbalance_ratio=1.509033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[58003, 108262, 102850, 472, 95992, 65331, 53944, 39434] avg=65536.0000 max=108262 max_rank=1 min=472 min_rank=3 imbalance_ratio=1.651947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] avg=65536.0000 max=66194 max_rank=3 min=61805 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42068 max_load_reduction_pct=0.388576 moved_to_replicas=215092 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] avg=65536.0000 max=66194 max_rank=3 min=61805 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42068 max_load_reduction_pct=0.388576 moved_to_replicas=215092 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[65349, 80331, 19066, 65857, 64211, 98896, 32879, 97699] avg=65536.0000 max=98896 max_rank=5 min=19066 min_rank=2 imbalance_ratio=1.509033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] avg=65536.0000 max=66194 max_rank=3 min=61805 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42068 max_load_reduction_pct=0.388576 moved_to_replicas=215092 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] avg=65536.0000 max=66192 max_rank=5 min=64211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32704 max_load_reduction_pct=0.330691 moved_to_replicas=155891 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[61805, 66192, 66192, 66194, 66192, 65331, 66192, 66190] avg=65536.0000 max=66194 max_rank=3 min=61805 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=42068 max_load_reduction_pct=0.388576 moved_to_replicas=215092 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[65349, 80331, 19066, 65857, 64211, 98896, 32879, 97699] avg=65536.0000 max=98896 max_rank=5 min=19066 min_rank=2 imbalance_ratio=1.509033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[65349, 80331, 19066, 65857, 64211, 98896, 32879, 97699] avg=65536.0000 max=98896 max_rank=5 min=19066 min_rank=2 imbalance_ratio=1.509033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] avg=65536.0000 max=66192 max_rank=5 min=64211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32704 max_load_reduction_pct=0.330691 moved_to_replicas=155891 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[65349, 80331, 19066, 65857, 64211, 98896, 32879, 97699] avg=65536.0000 max=98896 max_rank=5 min=19066 min_rank=2 imbalance_ratio=1.509033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[65349, 80331, 19066, 65857, 64211, 98896, 32879, 97699] avg=65536.0000 max=98896 max_rank=5 min=19066 min_rank=2 imbalance_ratio=1.509033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[65349, 80331, 19066, 65857, 64211, 98896, 32879, 97699] avg=65536.0000 max=98896 max_rank=5 min=19066 min_rank=2 imbalance_ratio=1.509033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] avg=65536.0000 max=66192 max_rank=5 min=64211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32704 max_load_reduction_pct=0.330691 moved_to_replicas=155891 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65531, 35864, 32844, 98275, 130895, 98304, 60882, 1693] avg=65536.0000 max=130895 max_rank=4 min=1693 min_rank=7 imbalance_ratio=1.997299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[65349, 80331, 19066, 65857, 64211, 98896, 32879, 97699] avg=65536.0000 max=98896 max_rank=5 min=19066 min_rank=2 imbalance_ratio=1.509033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] avg=65536.0000 max=66192 max_rank=5 min=64211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32704 max_load_reduction_pct=0.330691 moved_to_replicas=155891 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] avg=65536.0000 max=66192 max_rank=5 min=64211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32704 max_load_reduction_pct=0.330691 moved_to_replicas=155891 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] avg=65536.0000 max=66192 max_rank=5 min=64211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32704 max_load_reduction_pct=0.330691 moved_to_replicas=155891 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] avg=65536.0000 max=66192 max_rank=5 min=64211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32704 max_load_reduction_pct=0.330691 moved_to_replicas=155891 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65531, 35864, 32844, 98275, 130895, 98304, 60882, 1693] avg=65536.0000 max=130895 max_rank=4 min=1693 min_rank=7 imbalance_ratio=1.997299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] avg=65536.0000 max=66192 max_rank=1 min=63016 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.494312 moved_to_replicas=257226 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[65349, 66191, 65910, 65857, 64211, 66192, 64386, 66192] avg=65536.0000 max=66192 max_rank=5 min=64211 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32704 max_load_reduction_pct=0.330691 moved_to_replicas=155891 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65531, 35864, 32844, 98275, 130895, 98304, 60882, 1693] avg=65536.0000 max=130895 max_rank=4 min=1693 min_rank=7 imbalance_ratio=1.997299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] avg=65536.0000 max=66192 max_rank=1 min=63016 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.494312 moved_to_replicas=257226 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65531, 35864, 32844, 98275, 130895, 98304, 60882, 1693] avg=65536.0000 max=130895 max_rank=4 min=1693 min_rank=7 imbalance_ratio=1.997299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65531, 35864, 32844, 98275, 130895, 98304, 60882, 1693] avg=65536.0000 max=130895 max_rank=4 min=1693 min_rank=7 imbalance_ratio=1.997299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65531, 35864, 32844, 98275, 130895, 98304, 60882, 1693] avg=65536.0000 max=130895 max_rank=4 min=1693 min_rank=7 imbalance_ratio=1.997299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65531, 35864, 32844, 98275, 130895, 98304, 60882, 1693] avg=65536.0000 max=130895 max_rank=4 min=1693 min_rank=7 imbalance_ratio=1.997299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] avg=65536.0000 max=66192 max_rank=1 min=63016 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.494312 moved_to_replicas=257226 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[17073, 98249, 129528, 65409, 457, 49721, 98306, 65545] avg=65536.0000 max=129528 max_rank=2 min=457 min_rank=4 imbalance_ratio=1.976440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65531, 35864, 32844, 98275, 130895, 98304, 60882, 1693] avg=65536.0000 max=130895 max_rank=4 min=1693 min_rank=7 imbalance_ratio=1.997299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] avg=65536.0000 max=66192 max_rank=1 min=63016 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.494312 moved_to_replicas=257226 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] avg=65536.0000 max=66192 max_rank=1 min=63016 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.494312 moved_to_replicas=257226 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] avg=65536.0000 max=66192 max_rank=1 min=63016 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.494312 moved_to_replicas=257226 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] avg=65536.0000 max=66192 max_rank=1 min=63016 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.494312 moved_to_replicas=257226 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[17073, 98249, 129528, 65409, 457, 49721, 98306, 65545] avg=65536.0000 max=129528 max_rank=2 min=457 min_rank=4 imbalance_ratio=1.976440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] avg=65536.0000 max=66192 max_rank=1 min=63232 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63336 max_load_reduction_pct=0.488975 moved_to_replicas=254745 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65531, 66192, 64781, 66192, 66192, 66192, 63016, 66192] avg=65536.0000 max=66192 max_rank=1 min=63016 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.494312 moved_to_replicas=257226 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[17073, 98249, 129528, 65409, 457, 49721, 98306, 65545] avg=65536.0000 max=129528 max_rank=2 min=457 min_rank=4 imbalance_ratio=1.976440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] avg=65536.0000 max=66192 max_rank=1 min=63232 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63336 max_load_reduction_pct=0.488975 moved_to_replicas=254745 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[17073, 98249, 129528, 65409, 457, 49721, 98306, 65545] avg=65536.0000 max=129528 max_rank=2 min=457 min_rank=4 imbalance_ratio=1.976440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[17073, 98249, 129528, 65409, 457, 49721, 98306, 65545] avg=65536.0000 max=129528 max_rank=2 min=457 min_rank=4 imbalance_ratio=1.976440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[17073, 98249, 129528, 65409, 457, 49721, 98306, 65545] avg=65536.0000 max=129528 max_rank=2 min=457 min_rank=4 imbalance_ratio=1.976440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[17073, 98249, 129528, 65409, 457, 49721, 98306, 65545] avg=65536.0000 max=129528 max_rank=2 min=457 min_rank=4 imbalance_ratio=1.976440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] avg=65536.0000 max=66192 max_rank=1 min=63232 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63336 max_load_reduction_pct=0.488975 moved_to_replicas=254745 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[93720, 97931, 107836, 66910, 58499, 1035, 32770, 65587] avg=65536.0000 max=107836 max_rank=2 min=1035 min_rank=5 imbalance_ratio=1.645447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[17073, 98249, 129528, 65409, 457, 49721, 98306, 65545] avg=65536.0000 max=129528 max_rank=2 min=457 min_rank=4 imbalance_ratio=1.976440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] avg=65536.0000 max=66192 max_rank=1 min=63232 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63336 max_load_reduction_pct=0.488975 moved_to_replicas=254745 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] avg=65536.0000 max=66192 max_rank=1 min=63232 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63336 max_load_reduction_pct=0.488975 moved_to_replicas=254745 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] avg=65536.0000 max=66192 max_rank=1 min=63232 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63336 max_load_reduction_pct=0.488975 moved_to_replicas=254745 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] avg=65536.0000 max=66192 max_rank=1 min=63232 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63336 max_load_reduction_pct=0.488975 moved_to_replicas=254745 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[93720, 97931, 107836, 66910, 58499, 1035, 32770, 65587] avg=65536.0000 max=107836 max_rank=2 min=1035 min_rank=5 imbalance_ratio=1.645447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] avg=65536.0000 max=66192 max_rank=0 min=62201 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41644 max_load_reduction_pct=0.386179 moved_to_replicas=204961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[66190, 66192, 66192, 65409, 65336, 63232, 66192, 65545] avg=65536.0000 max=66192 max_rank=1 min=63232 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63336 max_load_reduction_pct=0.488975 moved_to_replicas=254745 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[93720, 97931, 107836, 66910, 58499, 1035, 32770, 65587] avg=65536.0000 max=107836 max_rank=2 min=1035 min_rank=5 imbalance_ratio=1.645447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] avg=65536.0000 max=66192 max_rank=0 min=62201 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41644 max_load_reduction_pct=0.386179 moved_to_replicas=204961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[93720, 97931, 107836, 66910, 58499, 1035, 32770, 65587] avg=65536.0000 max=107836 max_rank=2 min=1035 min_rank=5 imbalance_ratio=1.645447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[93720, 97931, 107836, 66910, 58499, 1035, 32770, 65587] avg=65536.0000 max=107836 max_rank=2 min=1035 min_rank=5 imbalance_ratio=1.645447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[93720, 97931, 107836, 66910, 58499, 1035, 32770, 65587] avg=65536.0000 max=107836 max_rank=2 min=1035 min_rank=5 imbalance_ratio=1.645447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[93720, 97931, 107836, 66910, 58499, 1035, 32770, 65587] avg=65536.0000 max=107836 max_rank=2 min=1035 min_rank=5 imbalance_ratio=1.645447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32786, 43785, 433, 65559, 65535, 154741, 65993, 95456] avg=65536.0000 max=154741 max_rank=5 min=433 min_rank=2 imbalance_ratio=2.361160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] avg=65536.0000 max=66192 max_rank=0 min=62201 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41644 max_load_reduction_pct=0.386179 moved_to_replicas=204961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[93720, 97931, 107836, 66910, 58499, 1035, 32770, 65587] avg=65536.0000 max=107836 max_rank=2 min=1035 min_rank=5 imbalance_ratio=1.645447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] avg=65536.0000 max=66192 max_rank=0 min=62201 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41644 max_load_reduction_pct=0.386179 moved_to_replicas=204961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] avg=65536.0000 max=66192 max_rank=0 min=62201 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41644 max_load_reduction_pct=0.386179 moved_to_replicas=204961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] avg=65536.0000 max=66192 max_rank=0 min=62201 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41644 max_load_reduction_pct=0.386179 moved_to_replicas=204961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32786, 43785, 433, 65559, 65535, 154741, 65993, 95456] avg=65536.0000 max=154741 max_rank=5 min=433 min_rank=2 imbalance_ratio=2.361160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] avg=65536.0000 max=66192 max_rank=0 min=62201 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41644 max_load_reduction_pct=0.386179 moved_to_replicas=204961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] avg=65536.0000 max=66192 max_rank=5 min=63146 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=88549 max_load_reduction_pct=0.572240 moved_to_replicas=233238 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[66192, 66192, 66192, 66192, 62201, 65540, 66192, 65587] avg=65536.0000 max=66192 max_rank=0 min=62201 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41644 max_load_reduction_pct=0.386179 moved_to_replicas=204961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32786, 43785, 433, 65559, 65535, 154741, 65993, 95456] avg=65536.0000 max=154741 max_rank=5 min=433 min_rank=2 imbalance_ratio=2.361160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] avg=65536.0000 max=66192 max_rank=5 min=63146 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=88549 max_load_reduction_pct=0.572240 moved_to_replicas=233238 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32786, 43785, 433, 65559, 65535, 154741, 65993, 95456] avg=65536.0000 max=154741 max_rank=5 min=433 min_rank=2 imbalance_ratio=2.361160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32786, 43785, 433, 65559, 65535, 154741, 65993, 95456] avg=65536.0000 max=154741 max_rank=5 min=433 min_rank=2 imbalance_ratio=2.361160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32786, 43785, 433, 65559, 65535, 154741, 65993, 95456] avg=65536.0000 max=154741 max_rank=5 min=433 min_rank=2 imbalance_ratio=2.361160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32786, 43785, 433, 65559, 65535, 154741, 65993, 95456] avg=65536.0000 max=154741 max_rank=5 min=433 min_rank=2 imbalance_ratio=2.361160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65633, 65532, 32772, 31337, 37804, 94591, 98317] avg=65536.0000 max=98317 max_rank=7 min=31337 min_rank=4 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32786, 43785, 433, 65559, 65535, 154741, 65993, 95456] avg=65536.0000 max=154741 max_rank=5 min=433 min_rank=2 imbalance_ratio=2.361160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] avg=65536.0000 max=66192 max_rank=5 min=63146 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=88549 max_load_reduction_pct=0.572240 moved_to_replicas=233238 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] avg=65536.0000 max=66192 max_rank=5 min=63146 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=88549 max_load_reduction_pct=0.572240 moved_to_replicas=233238 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] avg=65536.0000 max=66192 max_rank=5 min=63146 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=88549 max_load_reduction_pct=0.572240 moved_to_replicas=233238 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] avg=65536.0000 max=66192 max_rank=5 min=63146 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=88549 max_load_reduction_pct=0.572240 moved_to_replicas=233238 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] avg=65536.0000 max=66192 max_rank=5 min=63146 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=88549 max_load_reduction_pct=0.572240 moved_to_replicas=233238 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65633, 65532, 32772, 31337, 37804, 94591, 98317] avg=65536.0000 max=98317 max_rank=7 min=31337 min_rank=4 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63473 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=182442 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65481, 66190, 63146, 65559, 65535, 66192, 65993, 66192] avg=65536.0000 max=66192 max_rank=5 min=63146 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=88549 max_load_reduction_pct=0.572240 moved_to_replicas=233238 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65633, 65532, 32772, 31337, 37804, 94591, 98317] avg=65536.0000 max=98317 max_rank=7 min=31337 min_rank=4 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63473 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=182442 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65633, 65532, 32772, 31337, 37804, 94591, 98317] avg=65536.0000 max=98317 max_rank=7 min=31337 min_rank=4 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65633, 65532, 32772, 31337, 37804, 94591, 98317] avg=65536.0000 max=98317 max_rank=7 min=31337 min_rank=4 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65633, 65532, 32772, 31337, 37804, 94591, 98317] avg=65536.0000 max=98317 max_rank=7 min=31337 min_rank=4 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65633, 65532, 32772, 31337, 37804, 94591, 98317] avg=65536.0000 max=98317 max_rank=7 min=31337 min_rank=4 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[98269, 16481, 32780, 98333, 98286, 67948, 79402, 32789] avg=65536.0000 max=98333 max_rank=3 min=16481 min_rank=1 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[98302, 65633, 65532, 32772, 31337, 37804, 94591, 98317] avg=65536.0000 max=98317 max_rank=7 min=31337 min_rank=4 imbalance_ratio=1.500198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63473 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=182442 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63473 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=182442 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63473 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=182442 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63473 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=182442 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63473 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=182442 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[98269, 16481, 32780, 98333, 98286, 67948, 79402, 32789] avg=65536.0000 max=98333 max_rank=3 min=16481 min_rank=1 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] avg=65536.0000 max=66192 max_rank=0 min=63588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=219408 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[66192, 65633, 65532, 64882, 63473, 66192, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=63473 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32125 max_load_reduction_pct=0.326749 moved_to_replicas=182442 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[98269, 16481, 32780, 98333, 98286, 67948, 79402, 32789] avg=65536.0000 max=98333 max_rank=3 min=16481 min_rank=1 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[98269, 16481, 32780, 98333, 98286, 67948, 79402, 32789] avg=65536.0000 max=98333 max_rank=3 min=16481 min_rank=1 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] avg=65536.0000 max=66192 max_rank=0 min=63588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=219408 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[98269, 16481, 32780, 98333, 98286, 67948, 79402, 32789] avg=65536.0000 max=98333 max_rank=3 min=16481 min_rank=1 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[98269, 16481, 32780, 98333, 98286, 67948, 79402, 32789] avg=65536.0000 max=98333 max_rank=3 min=16481 min_rank=1 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[98269, 16481, 32780, 98333, 98286, 67948, 79402, 32789] avg=65536.0000 max=98333 max_rank=3 min=16481 min_rank=1 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[65326, 32845, 110488, 104915, 3, 145203, 32702, 32806] avg=65536.0000 max=145203 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.215622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[98269, 16481, 32780, 98333, 98286, 67948, 79402, 32789] avg=65536.0000 max=98333 max_rank=3 min=16481 min_rank=1 imbalance_ratio=1.500443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] avg=65536.0000 max=66192 max_rank=0 min=63588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=219408 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] avg=65536.0000 max=66192 max_rank=0 min=63588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=219408 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] avg=65536.0000 max=66192 max_rank=0 min=63588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=219408 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] avg=65536.0000 max=66192 max_rank=0 min=63588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=219408 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] avg=65536.0000 max=66192 max_rank=0 min=63588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=219408 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[65326, 32845, 110488, 104915, 3, 145203, 32702, 32806] avg=65536.0000 max=145203 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.215622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] avg=65536.0000 max=66194 max_rank=2 min=63150 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79009 max_load_reduction_pct=0.544128 moved_to_replicas=324053 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[66192, 63588, 64874, 66192, 66192, 66192, 66192, 64866] avg=65536.0000 max=66192 max_rank=0 min=63588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32141 max_load_reduction_pct=0.326859 moved_to_replicas=219408 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[65326, 32845, 110488, 104915, 3, 145203, 32702, 32806] avg=65536.0000 max=145203 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.215622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[65326, 32845, 110488, 104915, 3, 145203, 32702, 32806] avg=65536.0000 max=145203 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.215622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] avg=65536.0000 max=66194 max_rank=2 min=63150 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79009 max_load_reduction_pct=0.544128 moved_to_replicas=324053 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[65326, 32845, 110488, 104915, 3, 145203, 32702, 32806] avg=65536.0000 max=145203 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.215622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[65326, 32845, 110488, 104915, 3, 145203, 32702, 32806] avg=65536.0000 max=145203 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.215622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[65326, 32845, 110488, 104915, 3, 145203, 32702, 32806] avg=65536.0000 max=145203 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.215622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[9406, 32804, 1967, 98274, 54657, 163708, 97946, 65526] avg=65536.0000 max=163708 max_rank=5 min=1967 min_rank=2 imbalance_ratio=2.497986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[65326, 32845, 110488, 104915, 3, 145203, 32702, 32806] avg=65536.0000 max=145203 max_rank=5 min=3 min_rank=4 imbalance_ratio=2.215622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] avg=65536.0000 max=66194 max_rank=2 min=63150 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79009 max_load_reduction_pct=0.544128 moved_to_replicas=324053 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] avg=65536.0000 max=66194 max_rank=2 min=63150 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79009 max_load_reduction_pct=0.544128 moved_to_replicas=324053 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] avg=65536.0000 max=66194 max_rank=2 min=63150 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79009 max_load_reduction_pct=0.544128 moved_to_replicas=324053 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] avg=65536.0000 max=66194 max_rank=2 min=63150 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79009 max_load_reduction_pct=0.544128 moved_to_replicas=324053 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[9406, 32804, 1967, 98274, 54657, 163708, 97946, 65526] avg=65536.0000 max=163708 max_rank=5 min=1967 min_rank=2 imbalance_ratio=2.497986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] avg=65536.0000 max=66194 max_rank=2 min=63150 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79009 max_load_reduction_pct=0.544128 moved_to_replicas=324053 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] avg=65536.0000 max=66193 max_rank=0 min=63010 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=97515 max_load_reduction_pct=0.595664 moved_to_replicas=321519 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[65326, 66191, 66194, 66193, 63150, 66194, 65468, 65572] avg=65536.0000 max=66194 max_rank=2 min=63150 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=79009 max_load_reduction_pct=0.544128 moved_to_replicas=324053 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[9406, 32804, 1967, 98274, 54657, 163708, 97946, 65526] avg=65536.0000 max=163708 max_rank=5 min=1967 min_rank=2 imbalance_ratio=2.497986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[9406, 32804, 1967, 98274, 54657, 163708, 97946, 65526] avg=65536.0000 max=163708 max_rank=5 min=1967 min_rank=2 imbalance_ratio=2.497986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] avg=65536.0000 max=66193 max_rank=0 min=63010 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=97515 max_load_reduction_pct=0.595664 moved_to_replicas=321519 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[9406, 32804, 1967, 98274, 54657, 163708, 97946, 65526] avg=65536.0000 max=163708 max_rank=5 min=1967 min_rank=2 imbalance_ratio=2.497986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[9406, 32804, 1967, 98274, 54657, 163708, 97946, 65526] avg=65536.0000 max=163708 max_rank=5 min=1967 min_rank=2 imbalance_ratio=2.497986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] avg=65536.0000 max=66193 max_rank=0 min=63010 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=97515 max_load_reduction_pct=0.595664 moved_to_replicas=321519 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[9406, 32804, 1967, 98274, 54657, 163708, 97946, 65526] avg=65536.0000 max=163708 max_rank=5 min=1967 min_rank=2 imbalance_ratio=2.497986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[98332, 65477, 77715, 33169, 882, 65541, 98275, 84897] avg=65536.0000 max=98332 max_rank=0 min=882 min_rank=4 imbalance_ratio=1.500427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[9406, 32804, 1967, 98274, 54657, 163708, 97946, 65526] avg=65536.0000 max=163708 max_rank=5 min=1967 min_rank=2 imbalance_ratio=2.497986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] avg=65536.0000 max=66193 max_rank=0 min=63010 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=97515 max_load_reduction_pct=0.595664 moved_to_replicas=321519 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] avg=65536.0000 max=66193 max_rank=0 min=63010 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=97515 max_load_reduction_pct=0.595664 moved_to_replicas=321519 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] avg=65536.0000 max=66193 max_rank=0 min=63010 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=97515 max_load_reduction_pct=0.595664 moved_to_replicas=321519 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] avg=65536.0000 max=66193 max_rank=0 min=63010 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=97515 max_load_reduction_pct=0.595664 moved_to_replicas=321519 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[98332, 65477, 77715, 33169, 882, 65541, 98275, 84897] avg=65536.0000 max=98332 max_rank=0 min=882 min_rank=4 imbalance_ratio=1.500427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63396 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32138 max_load_reduction_pct=0.326832 moved_to_replicas=188407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[66193, 64791, 66192, 66192, 63010, 66192, 66192, 65526] avg=65536.0000 max=66193 max_rank=0 min=63010 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=97515 max_load_reduction_pct=0.595664 moved_to_replicas=321519 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[98332, 65477, 77715, 33169, 882, 65541, 98275, 84897] avg=65536.0000 max=98332 max_rank=0 min=882 min_rank=4 imbalance_ratio=1.500427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[98332, 65477, 77715, 33169, 882, 65541, 98275, 84897] avg=65536.0000 max=98332 max_rank=0 min=882 min_rank=4 imbalance_ratio=1.500427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[98332, 65477, 77715, 33169, 882, 65541, 98275, 84897] avg=65536.0000 max=98332 max_rank=0 min=882 min_rank=4 imbalance_ratio=1.500427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63396 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32138 max_load_reduction_pct=0.326832 moved_to_replicas=188407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63396 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32138 max_load_reduction_pct=0.326832 moved_to_replicas=188407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[98332, 65477, 77715, 33169, 882, 65541, 98275, 84897] avg=65536.0000 max=98332 max_rank=0 min=882 min_rank=4 imbalance_ratio=1.500427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[98332, 65477, 77715, 33169, 882, 65541, 98275, 84897] avg=65536.0000 max=98332 max_rank=0 min=882 min_rank=4 imbalance_ratio=1.500427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[86467, 32765, 98219, 32776, 126869, 44881, 65536, 36775] avg=65536.0000 max=126869 max_rank=4 min=32765 min_rank=1 imbalance_ratio=1.935867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[98332, 65477, 77715, 33169, 882, 65541, 98275, 84897] avg=65536.0000 max=98332 max_rank=0 min=882 min_rank=4 imbalance_ratio=1.500427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63396 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32138 max_load_reduction_pct=0.326832 moved_to_replicas=188407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63396 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32138 max_load_reduction_pct=0.326832 moved_to_replicas=188407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63396 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32138 max_load_reduction_pct=0.326832 moved_to_replicas=188407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[86467, 32765, 98219, 32776, 126869, 44881, 65536, 36775] avg=65536.0000 max=126869 max_rank=4 min=32765 min_rank=1 imbalance_ratio=1.935867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63396 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32138 max_load_reduction_pct=0.326832 moved_to_replicas=188407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[66192, 65477, 66191, 63396, 65105, 65541, 66192, 66194] avg=65536.0000 max=66194 max_rank=7 min=63396 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32138 max_load_reduction_pct=0.326832 moved_to_replicas=188407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] avg=65536.0000 max=66194 max_rank=4 min=62259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60675 max_load_reduction_pct=0.478249 moved_to_replicas=227129 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[86467, 32765, 98219, 32776, 126869, 44881, 65536, 36775] avg=65536.0000 max=126869 max_rank=4 min=32765 min_rank=1 imbalance_ratio=1.935867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[86467, 32765, 98219, 32776, 126869, 44881, 65536, 36775] avg=65536.0000 max=126869 max_rank=4 min=32765 min_rank=1 imbalance_ratio=1.935867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[86467, 32765, 98219, 32776, 126869, 44881, 65536, 36775] avg=65536.0000 max=126869 max_rank=4 min=32765 min_rank=1 imbalance_ratio=1.935867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] avg=65536.0000 max=66194 max_rank=4 min=62259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60675 max_load_reduction_pct=0.478249 moved_to_replicas=227129 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] avg=65536.0000 max=66194 max_rank=4 min=62259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60675 max_load_reduction_pct=0.478249 moved_to_replicas=227129 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[86467, 32765, 98219, 32776, 126869, 44881, 65536, 36775] avg=65536.0000 max=126869 max_rank=4 min=32765 min_rank=1 imbalance_ratio=1.935867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[86467, 32765, 98219, 32776, 126869, 44881, 65536, 36775] avg=65536.0000 max=126869 max_rank=4 min=32765 min_rank=1 imbalance_ratio=1.935867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[86467, 32765, 98219, 32776, 126869, 44881, 65536, 36775] avg=65536.0000 max=126869 max_rank=4 min=32765 min_rank=1 imbalance_ratio=1.935867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[97690, 65431, 65554, 66096, 32771, 32952, 65508, 98286] avg=65536.0000 max=98286 max_rank=7 min=32771 min_rank=4 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] avg=65536.0000 max=66194 max_rank=4 min=62259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60675 max_load_reduction_pct=0.478249 moved_to_replicas=227129 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] avg=65536.0000 max=66194 max_rank=4 min=62259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60675 max_load_reduction_pct=0.478249 moved_to_replicas=227129 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] avg=65536.0000 max=66194 max_rank=4 min=62259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60675 max_load_reduction_pct=0.478249 moved_to_replicas=227129 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] avg=65536.0000 max=66194 max_rank=4 min=62259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60675 max_load_reduction_pct=0.478249 moved_to_replicas=227129 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[97690, 65431, 65554, 66096, 32771, 32952, 65508, 98286] avg=65536.0000 max=98286 max_rank=7 min=32771 min_rank=4 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[66190, 65531, 66192, 62259, 66194, 66194, 65536, 66192] avg=65536.0000 max=66194 max_rank=4 min=62259 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=60675 max_load_reduction_pct=0.478249 moved_to_replicas=227129 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] avg=65536.0000 max=66192 max_rank=0 min=64450 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32094 max_load_reduction_pct=0.326537 moved_to_replicas=126804 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[97690, 65431, 65554, 66096, 32771, 32952, 65508, 98286] avg=65536.0000 max=98286 max_rank=7 min=32771 min_rank=4 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[97690, 65431, 65554, 66096, 32771, 32952, 65508, 98286] avg=65536.0000 max=98286 max_rank=7 min=32771 min_rank=4 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[97690, 65431, 65554, 66096, 32771, 32952, 65508, 98286] avg=65536.0000 max=98286 max_rank=7 min=32771 min_rank=4 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] avg=65536.0000 max=66192 max_rank=0 min=64450 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32094 max_load_reduction_pct=0.326537 moved_to_replicas=126804 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] avg=65536.0000 max=66192 max_rank=0 min=64450 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32094 max_load_reduction_pct=0.326537 moved_to_replicas=126804 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[97690, 65431, 65554, 66096, 32771, 32952, 65508, 98286] avg=65536.0000 max=98286 max_rank=7 min=32771 min_rank=4 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] avg=65536.0000 max=66192 max_rank=0 min=64450 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32094 max_load_reduction_pct=0.326537 moved_to_replicas=126804 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[97690, 65431, 65554, 66096, 32771, 32952, 65508, 98286] avg=65536.0000 max=98286 max_rank=7 min=32771 min_rank=4 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[97690, 65431, 65554, 66096, 32771, 32952, 65508, 98286] avg=65536.0000 max=98286 max_rank=7 min=32771 min_rank=4 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] avg=65536.0000 max=66192 max_rank=0 min=64450 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32094 max_load_reduction_pct=0.326537 moved_to_replicas=126804 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] avg=65536.0000 max=66192 max_rank=0 min=64450 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32094 max_load_reduction_pct=0.326537 moved_to_replicas=126804 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] avg=65536.0000 max=66192 max_rank=0 min=64450 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32094 max_load_reduction_pct=0.326537 moved_to_replicas=126804 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[66192, 65431, 65554, 66096, 64865, 64450, 65508, 66192] avg=65536.0000 max=66192 max_rank=0 min=64450 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32094 max_load_reduction_pct=0.326537 moved_to_replicas=126804 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=2092863 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=510848 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=2100328 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=504192 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=2101324 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=501760 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=2080694 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=521472 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=2102508 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=499200 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=2101126 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=501376 peak_demand=66195 required_capacity=1.010056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=2100841 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=502272 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=2097532 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=503552 peak_demand=66194 required_capacity=1.010040 + [2026-08-30 05:14:13] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 535.5 | throughput per GPU (TFLOP/s/GPU): 622.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.994877E+00 | loss scale: 1.0 | grad norm: 5.228 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=81920 expected_recv=130169 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=81920 expected_recv=126911 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=81920 expected_recv=130350 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=81920 expected_recv=132386 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131608 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=81920 expected_recv=130169 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=81920 expected_recv=126911 actual_valid_recv=128256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=81920 expected_recv=130350 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=81920 expected_recv=132386 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=128256 slack_slots=35584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131608 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=81920 expected_recv=126154 expected_overflow=0 exact_recv_per_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=81920 expected_recv=126154 actual_valid_recv=128000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=81920 expected_recv=128119 expected_overflow=0 exact_recv_per_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=128000 slack_slots=35840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=81920 expected_recv=128119 actual_valid_recv=129920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=129920 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=134144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=81920 expected_recv=129474 expected_overflow=0 exact_recv_per_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=134144 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=81920 expected_recv=129474 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=81920 expected_recv=132381 expected_overflow=0 exact_recv_per_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131105 expected_overflow=0 exact_recv_per_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=81920 expected_recv=129920 expected_overflow=0 exact_recv_per_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=81920 expected_recv=132381 actual_valid_recv=134400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131105 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=81920 expected_recv=128543 expected_overflow=0 exact_recv_per_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=81920 expected_recv=129920 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=134400 slack_slots=29440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=81920 expected_recv=128543 actual_valid_recv=130048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=81920 expected_recv=130081 expected_overflow=0 exact_recv_per_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=130048 slack_slots=33792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=81920 expected_recv=130081 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=81920 expected_recv=130614 expected_overflow=0 exact_recv_per_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=81920 expected_recv=129181 expected_overflow=0 exact_recv_per_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=81920 expected_recv=130614 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=81920 expected_recv=129181 actual_valid_recv=130816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=81920 expected_recv=129164 expected_overflow=0 exact_recv_per_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=130816 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=81920 expected_recv=129164 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130109 expected_overflow=0 exact_recv_per_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130109 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130832 expected_overflow=0 exact_recv_per_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=81920 expected_recv=128492 expected_overflow=0 exact_recv_per_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130832 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=81920 expected_recv=131103 expected_overflow=0 exact_recv_per_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=81920 expected_recv=128492 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=81920 expected_recv=131103 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130888 expected_overflow=0 exact_recv_per_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131090 expected_overflow=0 exact_recv_per_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130888 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131090 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=81920 expected_recv=128362 expected_overflow=0 exact_recv_per_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=81920 expected_recv=127204 expected_overflow=0 exact_recv_per_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=81920 expected_recv=128362 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=81920 expected_recv=127204 actual_valid_recv=128512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=129792 slack_slots=34048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=128512 slack_slots=35328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=81920 expected_recv=130563 expected_overflow=0 exact_recv_per_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=81920 expected_recv=130563 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=81920 expected_recv=127762 expected_overflow=0 exact_recv_per_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=81920 expected_recv=127762 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=81920 expected_recv=128333 expected_overflow=0 exact_recv_per_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=81920 expected_recv=128333 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=81920 expected_recv=131098 expected_overflow=0 exact_recv_per_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=129792 slack_slots=34048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=81920 expected_recv=129901 expected_overflow=0 exact_recv_per_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=81920 expected_recv=131098 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=81920 expected_recv=129901 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=81920 expected_recv=129936 expected_overflow=0 exact_recv_per_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=81920 expected_recv=129936 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=81920 expected_recv=128107 expected_overflow=0 exact_recv_per_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=81920 expected_recv=132387 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=81920 expected_recv=128107 actual_valid_recv=129536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131488 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=81920 expected_recv=132387 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=129536 slack_slots=34304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=81920 expected_recv=132072 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131488 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=81920 expected_recv=124449 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=81920 expected_recv=132072 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=81920 expected_recv=124449 actual_valid_recv=125696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=125696 slack_slots=38144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131030 expected_overflow=0 exact_recv_per_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=81920 expected_recv=132383 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131030 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131999 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=81920 expected_recv=132383 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131999 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=81920 expected_recv=130688 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=81920 expected_recv=130688 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=81920 expected_recv=129780 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=81920 expected_recv=129780 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=81920 expected_recv=126574 expected_overflow=0 exact_recv_per_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=81920 expected_recv=126574 actual_valid_recv=127872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=81920 expected_recv=129983 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=81920 expected_recv=125192 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=127872 slack_slots=35968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=81920 expected_recv=131480 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=81920 expected_recv=129983 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=81920 expected_recv=125192 actual_valid_recv=126848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=81920 expected_recv=131480 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=126848 slack_slots=36992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=81920 expected_recv=127558 expected_overflow=0 exact_recv_per_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=81920 expected_recv=127558 actual_valid_recv=129152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=129152 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=81920 expected_recv=129922 expected_overflow=0 exact_recv_per_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131905 expected_overflow=0 exact_recv_per_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=81920 expected_recv=130313 expected_overflow=0 exact_recv_per_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=81920 expected_recv=129922 actual_valid_recv=130944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131905 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=81920 expected_recv=130313 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=130944 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131726 expected_overflow=0 exact_recv_per_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133888 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=81920 expected_recv=127619 expected_overflow=0 exact_recv_per_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=81920 expected_recv=131190 expected_overflow=0 exact_recv_per_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=81920 expected_recv=127847 expected_overflow=0 exact_recv_per_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131726 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=81920 expected_recv=127619 actual_valid_recv=128640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=81920 expected_recv=131190 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=81920 expected_recv=127847 actual_valid_recv=129792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=134016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=128640 slack_slots=35200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=129792 slack_slots=34048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=134016 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=81920 expected_recv=130761 expected_overflow=0 exact_recv_per_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131998 expected_overflow=0 exact_recv_per_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=81920 expected_recv=130761 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131998 actual_valid_recv=133248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133248 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131021 expected_overflow=0 exact_recv_per_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=133376 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=81920 expected_recv=129402 expected_overflow=0 exact_recv_per_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131021 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=81920 expected_recv=128967 expected_overflow=0 exact_recv_per_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=81920 expected_recv=129402 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=81920 expected_recv=128967 actual_valid_recv=130560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131020 expected_overflow=0 exact_recv_per_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=81920 expected_recv=128677 expected_overflow=0 exact_recv_per_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=81920 expected_recv=130973 expected_overflow=0 exact_recv_per_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130560 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131020 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131022 expected_overflow=0 exact_recv_per_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=81920 expected_recv=130973 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=81920 expected_recv=128677 actual_valid_recv=130432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=130432 slack_slots=33408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131022 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=81920 expected_recv=132382 expected_overflow=0 exact_recv_per_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=81920 expected_recv=132382 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=81920 expected_recv=129008 expected_overflow=0 exact_recv_per_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=81920 expected_recv=129008 actual_valid_recv=130176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=81920 expected_recv=125488 expected_overflow=0 exact_recv_per_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=81920 expected_recv=132385 expected_overflow=0 exact_recv_per_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131896 expected_overflow=0 exact_recv_per_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=130176 slack_slots=33664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=81920 expected_recv=125488 actual_valid_recv=126720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131062 expected_overflow=0 exact_recv_per_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=81920 expected_recv=132385 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131896 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=126720 slack_slots=37120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133632 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131062 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[68313, 79438, 61857, 77158, 65415, 59591, 52786, 59730] avg=65536.0000 max=79438 max_rank=1 min=52786 min_rank=6 imbalance_ratio=1.212128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=81920 expected_recv=132384 expected_overflow=0 exact_recv_per_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133760 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=81920 expected_recv=130596 expected_overflow=0 exact_recv_per_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=81920 expected_recv=132384 actual_valid_recv=133504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[68313, 79438, 61857, 77158, 65415, 59591, 52786, 59730] avg=65536.0000 max=79438 max_rank=1 min=52786 min_rank=6 imbalance_ratio=1.212128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[68313, 79438, 61857, 77158, 65415, 59591, 52786, 59730] avg=65536.0000 max=79438 max_rank=1 min=52786 min_rank=6 imbalance_ratio=1.212128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] avg=65536.0000 max=66193 max_rank=3 min=63978 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=13245 max_load_reduction_pct=0.166734 moved_to_replicas=56902 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[68313, 79438, 61857, 77158, 65415, 59591, 52786, 59730] avg=65536.0000 max=79438 max_rank=1 min=52786 min_rank=6 imbalance_ratio=1.212128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=133504 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=81920 expected_recv=130596 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[68313, 79438, 61857, 77158, 65415, 59591, 52786, 59730] avg=65536.0000 max=79438 max_rank=1 min=52786 min_rank=6 imbalance_ratio=1.212128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] avg=65536.0000 max=66193 max_rank=3 min=63978 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=13245 max_load_reduction_pct=0.166734 moved_to_replicas=56902 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] avg=65536.0000 max=66193 max_rank=3 min=63978 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=13245 max_load_reduction_pct=0.166734 moved_to_replicas=56902 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[68313, 79438, 61857, 77158, 65415, 59591, 52786, 59730] avg=65536.0000 max=79438 max_rank=1 min=52786 min_rank=6 imbalance_ratio=1.212128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] avg=65536.0000 max=66193 max_rank=3 min=63978 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=13245 max_load_reduction_pct=0.166734 moved_to_replicas=56902 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] avg=65536.0000 max=66193 max_rank=3 min=63978 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=13245 max_load_reduction_pct=0.166734 moved_to_replicas=56902 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[40457, 100477, 68900, 69758, 38779, 70752, 53818, 81347] avg=65536.0000 max=100477 max_rank=1 min=38779 min_rank=4 imbalance_ratio=1.533157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[68313, 79438, 61857, 77158, 65415, 59591, 52786, 59730] avg=65536.0000 max=79438 max_rank=1 min=52786 min_rank=6 imbalance_ratio=1.212128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[68313, 79438, 61857, 77158, 65415, 59591, 52786, 59730] avg=65536.0000 max=79438 max_rank=1 min=52786 min_rank=6 imbalance_ratio=1.212128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] avg=65536.0000 max=66193 max_rank=3 min=63978 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=13245 max_load_reduction_pct=0.166734 moved_to_replicas=56902 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[40457, 100477, 68900, 69758, 38779, 70752, 53818, 81347] avg=65536.0000 max=100477 max_rank=1 min=38779 min_rank=4 imbalance_ratio=1.533157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] avg=65536.0000 max=66192 max_rank=1 min=62486 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34285 max_load_reduction_pct=0.341222 moved_to_replicas=118183 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[40457, 100477, 68900, 69758, 38779, 70752, 53818, 81347] avg=65536.0000 max=100477 max_rank=1 min=38779 min_rank=4 imbalance_ratio=1.533157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[40457, 100477, 68900, 69758, 38779, 70752, 53818, 81347] avg=65536.0000 max=100477 max_rank=1 min=38779 min_rank=4 imbalance_ratio=1.533157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[40457, 100477, 68900, 69758, 38779, 70752, 53818, 81347] avg=65536.0000 max=100477 max_rank=1 min=38779 min_rank=4 imbalance_ratio=1.533157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] avg=65536.0000 max=66193 max_rank=3 min=63978 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=13245 max_load_reduction_pct=0.166734 moved_to_replicas=56902 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[66192, 66192, 63978, 66193, 65415, 66192, 66032, 64094] avg=65536.0000 max=66193 max_rank=3 min=63978 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=13245 max_load_reduction_pct=0.166734 moved_to_replicas=56902 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] avg=65536.0000 max=66192 max_rank=1 min=62486 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34285 max_load_reduction_pct=0.341222 moved_to_replicas=118183 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] avg=65536.0000 max=66192 max_rank=1 min=62486 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34285 max_load_reduction_pct=0.341222 moved_to_replicas=118183 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] avg=65536.0000 max=66192 max_rank=1 min=62486 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34285 max_load_reduction_pct=0.341222 moved_to_replicas=118183 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] avg=65536.0000 max=66192 max_rank=1 min=62486 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34285 max_load_reduction_pct=0.341222 moved_to_replicas=118183 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[40457, 100477, 68900, 69758, 38779, 70752, 53818, 81347] avg=65536.0000 max=100477 max_rank=1 min=38779 min_rank=4 imbalance_ratio=1.533157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32686, 95393, 98217, 58658, 109370, 32198, 65543, 32223] avg=65536.0000 max=109370 max_rank=4 min=32198 min_rank=5 imbalance_ratio=1.668854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[40457, 100477, 68900, 69758, 38779, 70752, 53818, 81347] avg=65536.0000 max=100477 max_rank=1 min=38779 min_rank=4 imbalance_ratio=1.533157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[40457, 100477, 68900, 69758, 38779, 70752, 53818, 81347] avg=65536.0000 max=100477 max_rank=1 min=38779 min_rank=4 imbalance_ratio=1.533157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32686, 95393, 98217, 58658, 109370, 32198, 65543, 32223] avg=65536.0000 max=109370 max_rank=4 min=32198 min_rank=5 imbalance_ratio=1.668854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] avg=65536.0000 max=66192 max_rank=1 min=62486 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34285 max_load_reduction_pct=0.341222 moved_to_replicas=118183 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] avg=65536.0000 max=66193 max_rank=4 min=64339 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=43177 max_load_reduction_pct=0.394779 moved_to_replicas=210393 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32686, 95393, 98217, 58658, 109370, 32198, 65543, 32223] avg=65536.0000 max=109370 max_rank=4 min=32198 min_rank=5 imbalance_ratio=1.668854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32686, 95393, 98217, 58658, 109370, 32198, 65543, 32223] avg=65536.0000 max=109370 max_rank=4 min=32198 min_rank=5 imbalance_ratio=1.668854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32686, 95393, 98217, 58658, 109370, 32198, 65543, 32223] avg=65536.0000 max=109370 max_rank=4 min=32198 min_rank=5 imbalance_ratio=1.668854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] avg=65536.0000 max=66192 max_rank=1 min=62486 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34285 max_load_reduction_pct=0.341222 moved_to_replicas=118183 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] avg=65536.0000 max=66193 max_rank=4 min=64339 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=43177 max_load_reduction_pct=0.394779 moved_to_replicas=210393 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[62486, 66192, 66192, 66192, 66191, 66190, 64654, 66191] avg=65536.0000 max=66192 max_rank=1 min=62486 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34285 max_load_reduction_pct=0.341222 moved_to_replicas=118183 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] avg=65536.0000 max=66193 max_rank=4 min=64339 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=43177 max_load_reduction_pct=0.394779 moved_to_replicas=210393 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] avg=65536.0000 max=66193 max_rank=4 min=64339 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=43177 max_load_reduction_pct=0.394779 moved_to_replicas=210393 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] avg=65536.0000 max=66193 max_rank=4 min=64339 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=43177 max_load_reduction_pct=0.394779 moved_to_replicas=210393 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32686, 95393, 98217, 58658, 109370, 32198, 65543, 32223] avg=65536.0000 max=109370 max_rank=4 min=32198 min_rank=5 imbalance_ratio=1.668854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32903, 66885, 130258, 13, 98305, 66905, 63641, 65378] avg=65536.0000 max=130258 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.987579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32686, 95393, 98217, 58658, 109370, 32198, 65543, 32223] avg=65536.0000 max=109370 max_rank=4 min=32198 min_rank=5 imbalance_ratio=1.668854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32903, 66885, 130258, 13, 98305, 66905, 63641, 65378] avg=65536.0000 max=130258 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.987579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32686, 95393, 98217, 58658, 109370, 32198, 65543, 32223] avg=65536.0000 max=109370 max_rank=4 min=32198 min_rank=5 imbalance_ratio=1.668854 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] avg=65536.0000 max=66193 max_rank=4 min=64339 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=43177 max_load_reduction_pct=0.394779 moved_to_replicas=210393 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] avg=65536.0000 max=66192 max_rank=1 min=64354 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64066 max_load_reduction_pct=0.491839 moved_to_replicas=195817 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32903, 66885, 130258, 13, 98305, 66905, 63641, 65378] avg=65536.0000 max=130258 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.987579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32903, 66885, 130258, 13, 98305, 66905, 63641, 65378] avg=65536.0000 max=130258 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.987579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32903, 66885, 130258, 13, 98305, 66905, 63641, 65378] avg=65536.0000 max=130258 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.987579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] avg=65536.0000 max=66193 max_rank=4 min=64339 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=43177 max_load_reduction_pct=0.394779 moved_to_replicas=210393 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] avg=65536.0000 max=66192 max_rank=1 min=64354 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64066 max_load_reduction_pct=0.491839 moved_to_replicas=195817 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[64711, 66192, 66192, 64339, 66193, 64928, 65543, 66190] avg=65536.0000 max=66193 max_rank=4 min=64339 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=43177 max_load_reduction_pct=0.394779 moved_to_replicas=210393 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] avg=65536.0000 max=66192 max_rank=1 min=64354 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64066 max_load_reduction_pct=0.491839 moved_to_replicas=195817 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] avg=65536.0000 max=66192 max_rank=1 min=64354 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64066 max_load_reduction_pct=0.491839 moved_to_replicas=195817 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] avg=65536.0000 max=66192 max_rank=1 min=64354 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64066 max_load_reduction_pct=0.491839 moved_to_replicas=195817 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[27340, 65318, 35134, 130544, 33100, 87532, 65588, 79732] avg=65536.0000 max=130544 max_rank=3 min=27340 min_rank=0 imbalance_ratio=1.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32903, 66885, 130258, 13, 98305, 66905, 63641, 65378] avg=65536.0000 max=130258 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.987579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32903, 66885, 130258, 13, 98305, 66905, 63641, 65378] avg=65536.0000 max=130258 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.987579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[27340, 65318, 35134, 130544, 33100, 87532, 65588, 79732] avg=65536.0000 max=130544 max_rank=3 min=27340 min_rank=0 imbalance_ratio=1.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32903, 66885, 130258, 13, 98305, 66905, 63641, 65378] avg=65536.0000 max=130258 max_rank=2 min=13 min_rank=3 imbalance_ratio=1.987579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[27340, 65318, 35134, 130544, 33100, 87532, 65588, 79732] avg=65536.0000 max=130544 max_rank=3 min=27340 min_rank=0 imbalance_ratio=1.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] avg=65536.0000 max=66193 max_rank=2 min=63918 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64351 max_load_reduction_pct=0.492945 moved_to_replicas=204958 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] avg=65536.0000 max=66192 max_rank=1 min=64354 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64066 max_load_reduction_pct=0.491839 moved_to_replicas=195817 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[27340, 65318, 35134, 130544, 33100, 87532, 65588, 79732] avg=65536.0000 max=130544 max_rank=3 min=27340 min_rank=0 imbalance_ratio=1.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[27340, 65318, 35134, 130544, 33100, 87532, 65588, 79732] avg=65536.0000 max=130544 max_rank=3 min=27340 min_rank=0 imbalance_ratio=1.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] avg=65536.0000 max=66192 max_rank=1 min=64354 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64066 max_load_reduction_pct=0.491839 moved_to_replicas=195817 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] avg=65536.0000 max=66193 max_rank=2 min=63918 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64351 max_load_reduction_pct=0.492945 moved_to_replicas=204958 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65016, 66192, 66192, 64772, 66192, 66192, 64354, 65378] avg=65536.0000 max=66192 max_rank=1 min=64354 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64066 max_load_reduction_pct=0.491839 moved_to_replicas=195817 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] avg=65536.0000 max=66193 max_rank=2 min=63918 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64351 max_load_reduction_pct=0.492945 moved_to_replicas=204958 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] avg=65536.0000 max=66193 max_rank=2 min=63918 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64351 max_load_reduction_pct=0.492945 moved_to_replicas=204958 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] avg=65536.0000 max=66193 max_rank=2 min=63918 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64351 max_load_reduction_pct=0.492945 moved_to_replicas=204958 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 58039, 67555, 129693, 86017, 7856, 44663] avg=65536.0000 max=129693 max_rank=4 min=7856 min_rank=6 imbalance_ratio=1.978958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[27340, 65318, 35134, 130544, 33100, 87532, 65588, 79732] avg=65536.0000 max=130544 max_rank=3 min=27340 min_rank=0 imbalance_ratio=1.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[27340, 65318, 35134, 130544, 33100, 87532, 65588, 79732] avg=65536.0000 max=130544 max_rank=3 min=27340 min_rank=0 imbalance_ratio=1.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 58039, 67555, 129693, 86017, 7856, 44663] avg=65536.0000 max=129693 max_rank=4 min=7856 min_rank=6 imbalance_ratio=1.978958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] avg=65536.0000 max=66193 max_rank=5 min=62863 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63500 max_load_reduction_pct=0.489618 moved_to_replicas=171710 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[27340, 65318, 35134, 130544, 33100, 87532, 65588, 79732] avg=65536.0000 max=130544 max_rank=3 min=27340 min_rank=0 imbalance_ratio=1.991943 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 58039, 67555, 129693, 86017, 7856, 44663] avg=65536.0000 max=129693 max_rank=4 min=7856 min_rank=6 imbalance_ratio=1.978958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] avg=65536.0000 max=66193 max_rank=2 min=63918 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64351 max_load_reduction_pct=0.492945 moved_to_replicas=204958 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 58039, 67555, 129693, 86017, 7856, 44663] avg=65536.0000 max=129693 max_rank=4 min=7856 min_rank=6 imbalance_ratio=1.978958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 58039, 67555, 129693, 86017, 7856, 44663] avg=65536.0000 max=129693 max_rank=4 min=7856 min_rank=6 imbalance_ratio=1.978958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] avg=65536.0000 max=66193 max_rank=2 min=63918 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64351 max_load_reduction_pct=0.492945 moved_to_replicas=204958 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] avg=65536.0000 max=66193 max_rank=5 min=62863 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63500 max_load_reduction_pct=0.489618 moved_to_replicas=171710 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] avg=65536.0000 max=66193 max_rank=5 min=62863 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63500 max_load_reduction_pct=0.489618 moved_to_replicas=171710 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[63918, 65318, 66193, 66192, 64696, 66191, 65588, 66192] avg=65536.0000 max=66193 max_rank=2 min=63918 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=64351 max_load_reduction_pct=0.492945 moved_to_replicas=204958 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] avg=65536.0000 max=66193 max_rank=5 min=62863 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63500 max_load_reduction_pct=0.489618 moved_to_replicas=171710 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] avg=65536.0000 max=66193 max_rank=5 min=62863 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63500 max_load_reduction_pct=0.489618 moved_to_replicas=171710 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[267, 129751, 68802, 98197, 32329, 34114, 93264, 67564] avg=65536.0000 max=129751 max_rank=1 min=267 min_rank=0 imbalance_ratio=1.979843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 58039, 67555, 129693, 86017, 7856, 44663] avg=65536.0000 max=129693 max_rank=4 min=7856 min_rank=6 imbalance_ratio=1.978958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 58039, 67555, 129693, 86017, 7856, 44663] avg=65536.0000 max=129693 max_rank=4 min=7856 min_rank=6 imbalance_ratio=1.978958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[267, 129751, 68802, 98197, 32329, 34114, 93264, 67564] avg=65536.0000 max=129751 max_rank=1 min=267 min_rank=0 imbalance_ratio=1.979843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] avg=65536.0000 max=66192 max_rank=1 min=63798 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63559 max_load_reduction_pct=0.489854 moved_to_replicas=252388 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[267, 129751, 68802, 98197, 32329, 34114, 93264, 67564] avg=65536.0000 max=129751 max_rank=1 min=267 min_rank=0 imbalance_ratio=1.979843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 58039, 67555, 129693, 86017, 7856, 44663] avg=65536.0000 max=129693 max_rank=4 min=7856 min_rank=6 imbalance_ratio=1.978958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[267, 129751, 68802, 98197, 32329, 34114, 93264, 67564] avg=65536.0000 max=129751 max_rank=1 min=267 min_rank=0 imbalance_ratio=1.979843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] avg=65536.0000 max=66193 max_rank=5 min=62863 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63500 max_load_reduction_pct=0.489618 moved_to_replicas=171710 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[267, 129751, 68802, 98197, 32329, 34114, 93264, 67564] avg=65536.0000 max=129751 max_rank=1 min=267 min_rank=0 imbalance_ratio=1.979843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] avg=65536.0000 max=66193 max_rank=5 min=62863 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63500 max_load_reduction_pct=0.489618 moved_to_replicas=171710 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] avg=65536.0000 max=66192 max_rank=1 min=63798 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63559 max_load_reduction_pct=0.489854 moved_to_replicas=252388 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] avg=65536.0000 max=66192 max_rank=1 min=63798 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63559 max_load_reduction_pct=0.489854 moved_to_replicas=252388 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65534, 64931, 62863, 66192, 66192, 66193, 66193, 66190] avg=65536.0000 max=66193 max_rank=5 min=62863 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=63500 max_load_reduction_pct=0.489618 moved_to_replicas=171710 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] avg=65536.0000 max=66192 max_rank=1 min=63798 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63559 max_load_reduction_pct=0.489854 moved_to_replicas=252388 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] avg=65536.0000 max=66192 max_rank=1 min=63798 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63559 max_load_reduction_pct=0.489854 moved_to_replicas=252388 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[90024, 65619, 130573, 37532, 46982, 32774, 32775, 88009] avg=65536.0000 max=130573 max_rank=2 min=32774 min_rank=5 imbalance_ratio=1.992386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[267, 129751, 68802, 98197, 32329, 34114, 93264, 67564] avg=65536.0000 max=129751 max_rank=1 min=267 min_rank=0 imbalance_ratio=1.979843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[267, 129751, 68802, 98197, 32329, 34114, 93264, 67564] avg=65536.0000 max=129751 max_rank=1 min=267 min_rank=0 imbalance_ratio=1.979843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[90024, 65619, 130573, 37532, 46982, 32774, 32775, 88009] avg=65536.0000 max=130573 max_rank=2 min=32774 min_rank=5 imbalance_ratio=1.992386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] avg=65536.0000 max=66194 max_rank=7 min=63972 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64379 max_load_reduction_pct=0.493050 moved_to_replicas=220162 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[90024, 65619, 130573, 37532, 46982, 32774, 32775, 88009] avg=65536.0000 max=130573 max_rank=2 min=32774 min_rank=5 imbalance_ratio=1.992386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[267, 129751, 68802, 98197, 32329, 34114, 93264, 67564] avg=65536.0000 max=129751 max_rank=1 min=267 min_rank=0 imbalance_ratio=1.979843 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[90024, 65619, 130573, 37532, 46982, 32774, 32775, 88009] avg=65536.0000 max=130573 max_rank=2 min=32774 min_rank=5 imbalance_ratio=1.992386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] avg=65536.0000 max=66192 max_rank=1 min=63798 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63559 max_load_reduction_pct=0.489854 moved_to_replicas=252388 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[90024, 65619, 130573, 37532, 46982, 32774, 32775, 88009] avg=65536.0000 max=130573 max_rank=2 min=32774 min_rank=5 imbalance_ratio=1.992386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] avg=65536.0000 max=66192 max_rank=1 min=63798 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63559 max_load_reduction_pct=0.489854 moved_to_replicas=252388 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] avg=65536.0000 max=66194 max_rank=7 min=63972 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64379 max_load_reduction_pct=0.493050 moved_to_replicas=220162 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] avg=65536.0000 max=66194 max_rank=7 min=63972 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64379 max_load_reduction_pct=0.493050 moved_to_replicas=220162 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] avg=65536.0000 max=66194 max_rank=7 min=63972 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64379 max_load_reduction_pct=0.493050 moved_to_replicas=220162 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[65036, 66192, 66192, 66192, 64496, 63798, 66190, 66192] avg=65536.0000 max=66192 max_rank=1 min=63798 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63559 max_load_reduction_pct=0.489854 moved_to_replicas=252388 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32894, 20905, 97829, 86720, 65517, 104920, 61930, 53573] avg=65536.0000 max=104920 max_rank=5 min=20905 min_rank=1 imbalance_ratio=1.600952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] avg=65536.0000 max=66194 max_rank=7 min=63972 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64379 max_load_reduction_pct=0.493050 moved_to_replicas=220162 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[90024, 65619, 130573, 37532, 46982, 32774, 32775, 88009] avg=65536.0000 max=130573 max_rank=2 min=32774 min_rank=5 imbalance_ratio=1.992386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[90024, 65619, 130573, 37532, 46982, 32774, 32775, 88009] avg=65536.0000 max=130573 max_rank=2 min=32774 min_rank=5 imbalance_ratio=1.992386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] avg=65536.0000 max=66194 max_rank=0 min=61930 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38726 max_load_reduction_pct=0.369100 moved_to_replicas=184714 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32894, 20905, 97829, 86720, 65517, 104920, 61930, 53573] avg=65536.0000 max=104920 max_rank=5 min=20905 min_rank=1 imbalance_ratio=1.600952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32894, 20905, 97829, 86720, 65517, 104920, 61930, 53573] avg=65536.0000 max=104920 max_rank=5 min=20905 min_rank=1 imbalance_ratio=1.600952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32894, 20905, 97829, 86720, 65517, 104920, 61930, 53573] avg=65536.0000 max=104920 max_rank=5 min=20905 min_rank=1 imbalance_ratio=1.600952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] avg=65536.0000 max=66194 max_rank=7 min=63972 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64379 max_load_reduction_pct=0.493050 moved_to_replicas=220162 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[90024, 65619, 130573, 37532, 46982, 32774, 32775, 88009] avg=65536.0000 max=130573 max_rank=2 min=32774 min_rank=5 imbalance_ratio=1.992386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32894, 20905, 97829, 86720, 65517, 104920, 61930, 53573] avg=65536.0000 max=104920 max_rank=5 min=20905 min_rank=1 imbalance_ratio=1.600952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] avg=65536.0000 max=66194 max_rank=7 min=63972 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64379 max_load_reduction_pct=0.493050 moved_to_replicas=220162 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] avg=65536.0000 max=66194 max_rank=0 min=61930 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38726 max_load_reduction_pct=0.369100 moved_to_replicas=184714 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] avg=65536.0000 max=66194 max_rank=0 min=61930 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38726 max_load_reduction_pct=0.369100 moved_to_replicas=184714 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] avg=65536.0000 max=66194 max_rank=0 min=61930 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38726 max_load_reduction_pct=0.369100 moved_to_replicas=184714 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[98340, 94136, 39494, 32777, 29939, 34897, 96376, 98329] avg=65536.0000 max=98340 max_rank=0 min=29939 min_rank=4 imbalance_ratio=1.500549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[66191, 65619, 66192, 63972, 66190, 65537, 64393, 66194] avg=65536.0000 max=66194 max_rank=7 min=63972 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64379 max_load_reduction_pct=0.493050 moved_to_replicas=220162 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] avg=65536.0000 max=66194 max_rank=0 min=61930 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38726 max_load_reduction_pct=0.369100 moved_to_replicas=184714 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32894, 20905, 97829, 86720, 65517, 104920, 61930, 53573] avg=65536.0000 max=104920 max_rank=5 min=20905 min_rank=1 imbalance_ratio=1.600952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32894, 20905, 97829, 86720, 65517, 104920, 61930, 53573] avg=65536.0000 max=104920 max_rank=5 min=20905 min_rank=1 imbalance_ratio=1.600952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63333 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=32147 max_load_reduction_pct=0.326896 moved_to_replicas=247063 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[98340, 94136, 39494, 32777, 29939, 34897, 96376, 98329] avg=65536.0000 max=98340 max_rank=0 min=29939 min_rank=4 imbalance_ratio=1.500549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[98340, 94136, 39494, 32777, 29939, 34897, 96376, 98329] avg=65536.0000 max=98340 max_rank=0 min=29939 min_rank=4 imbalance_ratio=1.500549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[98340, 94136, 39494, 32777, 29939, 34897, 96376, 98329] avg=65536.0000 max=98340 max_rank=0 min=29939 min_rank=4 imbalance_ratio=1.500549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] avg=65536.0000 max=66194 max_rank=0 min=61930 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38726 max_load_reduction_pct=0.369100 moved_to_replicas=184714 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[98340, 94136, 39494, 32777, 29939, 34897, 96376, 98329] avg=65536.0000 max=98340 max_rank=0 min=29939 min_rank=4 imbalance_ratio=1.500549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] avg=65536.0000 max=66194 max_rank=0 min=61930 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38726 max_load_reduction_pct=0.369100 moved_to_replicas=184714 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32894, 20905, 97829, 86720, 65517, 104920, 61930, 53573] avg=65536.0000 max=104920 max_rank=5 min=20905 min_rank=1 imbalance_ratio=1.600952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63333 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=32147 max_load_reduction_pct=0.326896 moved_to_replicas=247063 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63333 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=32147 max_load_reduction_pct=0.326896 moved_to_replicas=247063 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63333 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=32147 max_load_reduction_pct=0.326896 moved_to_replicas=247063 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[98863, 2507, 13253, 97576, 104759, 76460, 65579, 65291] avg=65536.0000 max=104759 max_rank=4 min=2507 min_rank=1 imbalance_ratio=1.598495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63333 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=32147 max_load_reduction_pct=0.326896 moved_to_replicas=247063 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[66194, 66192, 66192, 66191, 65517, 66192, 61930, 65880] avg=65536.0000 max=66194 max_rank=0 min=61930 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=38726 max_load_reduction_pct=0.369100 moved_to_replicas=184714 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[98340, 94136, 39494, 32777, 29939, 34897, 96376, 98329] avg=65536.0000 max=98340 max_rank=0 min=29939 min_rank=4 imbalance_ratio=1.500549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[98340, 94136, 39494, 32777, 29939, 34897, 96376, 98329] avg=65536.0000 max=98340 max_rank=0 min=29939 min_rank=4 imbalance_ratio=1.500549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] avg=65536.0000 max=66192 max_rank=0 min=62458 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=38567 max_load_reduction_pct=0.368150 moved_to_replicas=227381 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[98863, 2507, 13253, 97576, 104759, 76460, 65579, 65291] avg=65536.0000 max=104759 max_rank=4 min=2507 min_rank=1 imbalance_ratio=1.598495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[98863, 2507, 13253, 97576, 104759, 76460, 65579, 65291] avg=65536.0000 max=104759 max_rank=4 min=2507 min_rank=1 imbalance_ratio=1.598495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[98863, 2507, 13253, 97576, 104759, 76460, 65579, 65291] avg=65536.0000 max=104759 max_rank=4 min=2507 min_rank=1 imbalance_ratio=1.598495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63333 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=32147 max_load_reduction_pct=0.326896 moved_to_replicas=247063 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[98863, 2507, 13253, 97576, 104759, 76460, 65579, 65291] avg=65536.0000 max=104759 max_rank=4 min=2507 min_rank=1 imbalance_ratio=1.598495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63333 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=32147 max_load_reduction_pct=0.326896 moved_to_replicas=247063 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[98340, 94136, 39494, 32777, 29939, 34897, 96376, 98329] avg=65536.0000 max=98340 max_rank=0 min=29939 min_rank=4 imbalance_ratio=1.500549 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] avg=65536.0000 max=66192 max_rank=0 min=62458 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=38567 max_load_reduction_pct=0.368150 moved_to_replicas=227381 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] avg=65536.0000 max=66192 max_rank=0 min=62458 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=38567 max_load_reduction_pct=0.368150 moved_to_replicas=227381 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] avg=65536.0000 max=66192 max_rank=0 min=62458 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=38567 max_load_reduction_pct=0.368150 moved_to_replicas=227381 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[63440, 33258, 130427, 98353, 11, 133443, 32406, 32950] avg=65536.0000 max=133443 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.036179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] avg=65536.0000 max=66192 max_rank=0 min=62458 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=38567 max_load_reduction_pct=0.368150 moved_to_replicas=227381 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[66192, 66191, 66193, 64914, 63333, 65081, 66192, 66192] avg=65536.0000 max=66193 max_rank=2 min=63333 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=32147 max_load_reduction_pct=0.326896 moved_to_replicas=247063 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[98863, 2507, 13253, 97576, 104759, 76460, 65579, 65291] avg=65536.0000 max=104759 max_rank=4 min=2507 min_rank=1 imbalance_ratio=1.598495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[98863, 2507, 13253, 97576, 104759, 76460, 65579, 65291] avg=65536.0000 max=104759 max_rank=4 min=2507 min_rank=1 imbalance_ratio=1.598495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] avg=65536.0000 max=66192 max_rank=2 min=63913 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67251 max_load_reduction_pct=0.503968 moved_to_replicas=327186 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[63440, 33258, 130427, 98353, 11, 133443, 32406, 32950] avg=65536.0000 max=133443 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.036179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[63440, 33258, 130427, 98353, 11, 133443, 32406, 32950] avg=65536.0000 max=133443 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.036179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[63440, 33258, 130427, 98353, 11, 133443, 32406, 32950] avg=65536.0000 max=133443 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.036179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] avg=65536.0000 max=66192 max_rank=0 min=62458 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=38567 max_load_reduction_pct=0.368150 moved_to_replicas=227381 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] avg=65536.0000 max=66192 max_rank=0 min=62458 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=38567 max_load_reduction_pct=0.368150 moved_to_replicas=227381 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[63440, 33258, 130427, 98353, 11, 133443, 32406, 32950] avg=65536.0000 max=133443 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.036179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[98863, 2507, 13253, 97576, 104759, 76460, 65579, 65291] avg=65536.0000 max=104759 max_rank=4 min=2507 min_rank=1 imbalance_ratio=1.598495 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] avg=65536.0000 max=66192 max_rank=2 min=63913 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67251 max_load_reduction_pct=0.503968 moved_to_replicas=327186 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] avg=65536.0000 max=66192 max_rank=2 min=63913 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67251 max_load_reduction_pct=0.503968 moved_to_replicas=327186 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] avg=65536.0000 max=66192 max_rank=2 min=63913 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67251 max_load_reduction_pct=0.503968 moved_to_replicas=327186 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[214, 32886, 182, 97432, 93134, 161686, 73218, 65536] avg=65536.0000 max=161686 max_rank=5 min=182 min_rank=2 imbalance_ratio=2.467133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] avg=65536.0000 max=66192 max_rank=2 min=63913 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67251 max_load_reduction_pct=0.503968 moved_to_replicas=327186 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[66192, 66192, 62458, 66192, 66192, 66192, 65579, 65291] avg=65536.0000 max=66192 max_rank=0 min=62458 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=38567 max_load_reduction_pct=0.368150 moved_to_replicas=227381 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[63440, 33258, 130427, 98353, 11, 133443, 32406, 32950] avg=65536.0000 max=133443 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.036179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[63440, 33258, 130427, 98353, 11, 133443, 32406, 32950] avg=65536.0000 max=133443 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.036179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=0 min=63606 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=95494 max_load_reduction_pct=0.590614 moved_to_replicas=320568 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[214, 32886, 182, 97432, 93134, 161686, 73218, 65536] avg=65536.0000 max=161686 max_rank=5 min=182 min_rank=2 imbalance_ratio=2.467133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[214, 32886, 182, 97432, 93134, 161686, 73218, 65536] avg=65536.0000 max=161686 max_rank=5 min=182 min_rank=2 imbalance_ratio=2.467133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[214, 32886, 182, 97432, 93134, 161686, 73218, 65536] avg=65536.0000 max=161686 max_rank=5 min=182 min_rank=2 imbalance_ratio=2.467133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] avg=65536.0000 max=66192 max_rank=2 min=63913 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67251 max_load_reduction_pct=0.503968 moved_to_replicas=327186 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] avg=65536.0000 max=66192 max_rank=2 min=63913 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67251 max_load_reduction_pct=0.503968 moved_to_replicas=327186 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[214, 32886, 182, 97432, 93134, 161686, 73218, 65536] avg=65536.0000 max=161686 max_rank=5 min=182 min_rank=2 imbalance_ratio=2.467133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[63440, 33258, 130427, 98353, 11, 133443, 32406, 32950] avg=65536.0000 max=133443 max_rank=5 min=11 min_rank=4 imbalance_ratio=2.036179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=0 min=63606 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=95494 max_load_reduction_pct=0.590614 moved_to_replicas=320568 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=0 min=63606 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=95494 max_load_reduction_pct=0.590614 moved_to_replicas=320568 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=0 min=63606 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=95494 max_load_reduction_pct=0.590614 moved_to_replicas=320568 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[98262, 65486, 65806, 63883, 90, 65549, 98268, 66944] avg=65536.0000 max=98268 max_rank=6 min=90 min_rank=4 imbalance_ratio=1.499451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=0 min=63606 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=95494 max_load_reduction_pct=0.590614 moved_to_replicas=320568 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[63913, 64730, 66192, 66192, 66192, 66192, 65164, 65713] avg=65536.0000 max=66192 max_rank=2 min=63913 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=67251 max_load_reduction_pct=0.503968 moved_to_replicas=327186 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[214, 32886, 182, 97432, 93134, 161686, 73218, 65536] avg=65536.0000 max=161686 max_rank=5 min=182 min_rank=2 imbalance_ratio=2.467133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[214, 32886, 182, 97432, 93134, 161686, 73218, 65536] avg=65536.0000 max=161686 max_rank=5 min=182 min_rank=2 imbalance_ratio=2.467133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=64236 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32076 max_load_reduction_pct=0.326413 moved_to_replicas=131497 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[98262, 65486, 65806, 63883, 90, 65549, 98268, 66944] avg=65536.0000 max=98268 max_rank=6 min=90 min_rank=4 imbalance_ratio=1.499451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[98262, 65486, 65806, 63883, 90, 65549, 98268, 66944] avg=65536.0000 max=98268 max_rank=6 min=90 min_rank=4 imbalance_ratio=1.499451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[98262, 65486, 65806, 63883, 90, 65549, 98268, 66944] avg=65536.0000 max=98268 max_rank=6 min=90 min_rank=4 imbalance_ratio=1.499451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=0 min=63606 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=95494 max_load_reduction_pct=0.590614 moved_to_replicas=320568 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=0 min=63606 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=95494 max_load_reduction_pct=0.590614 moved_to_replicas=320568 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[98262, 65486, 65806, 63883, 90, 65549, 98268, 66944] avg=65536.0000 max=98268 max_rank=6 min=90 min_rank=4 imbalance_ratio=1.499451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=64236 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32076 max_load_reduction_pct=0.326413 moved_to_replicas=131497 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=64236 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32076 max_load_reduction_pct=0.326413 moved_to_replicas=131497 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[64653, 32769, 91485, 32799, 106286, 97999, 65522, 32775] avg=65536.0000 max=106286 max_rank=4 min=32769 min_rank=1 imbalance_ratio=1.621796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=64236 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32076 max_load_reduction_pct=0.326413 moved_to_replicas=131497 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[214, 32886, 182, 97432, 93134, 161686, 73218, 65536] avg=65536.0000 max=161686 max_rank=5 min=182 min_rank=2 imbalance_ratio=2.467133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=64236 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32076 max_load_reduction_pct=0.326413 moved_to_replicas=131497 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[98262, 65486, 65806, 63883, 90, 65549, 98268, 66944] avg=65536.0000 max=98268 max_rank=6 min=90 min_rank=4 imbalance_ratio=1.499451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[98262, 65486, 65806, 63883, 90, 65549, 98268, 66944] avg=65536.0000 max=98268 max_rank=6 min=90 min_rank=4 imbalance_ratio=1.499451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] avg=65536.0000 max=66193 max_rank=4 min=64606 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=40093 max_load_reduction_pct=0.377218 moved_to_replicas=194304 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[64653, 32769, 91485, 32799, 106286, 97999, 65522, 32775] avg=65536.0000 max=106286 max_rank=4 min=32769 min_rank=1 imbalance_ratio=1.621796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[66192, 63606, 64186, 66192, 66192, 66192, 66192, 65536] avg=65536.0000 max=66192 max_rank=0 min=63606 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=95494 max_load_reduction_pct=0.590614 moved_to_replicas=320568 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[64653, 32769, 91485, 32799, 106286, 97999, 65522, 32775] avg=65536.0000 max=106286 max_rank=4 min=32769 min_rank=1 imbalance_ratio=1.621796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[64653, 32769, 91485, 32799, 106286, 97999, 65522, 32775] avg=65536.0000 max=106286 max_rank=4 min=32769 min_rank=1 imbalance_ratio=1.621796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=64236 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32076 max_load_reduction_pct=0.326413 moved_to_replicas=131497 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=64236 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32076 max_load_reduction_pct=0.326413 moved_to_replicas=131497 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[64653, 32769, 91485, 32799, 106286, 97999, 65522, 32775] avg=65536.0000 max=106286 max_rank=4 min=32769 min_rank=1 imbalance_ratio=1.621796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] avg=65536.0000 max=66193 max_rank=4 min=64606 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=40093 max_load_reduction_pct=0.377218 moved_to_replicas=194304 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] avg=65536.0000 max=66193 max_rank=4 min=64606 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=40093 max_load_reduction_pct=0.377218 moved_to_replicas=194304 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[94677, 34212, 65704, 98254, 32779, 33575, 65522, 99565] avg=65536.0000 max=99565 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.519241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] avg=65536.0000 max=66193 max_rank=4 min=64606 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=40093 max_load_reduction_pct=0.377218 moved_to_replicas=194304 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[98262, 65486, 65806, 63883, 90, 65549, 98268, 66944] avg=65536.0000 max=98268 max_rank=6 min=90 min_rank=4 imbalance_ratio=1.499451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] avg=65536.0000 max=66193 max_rank=4 min=64606 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=40093 max_load_reduction_pct=0.377218 moved_to_replicas=194304 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[64653, 32769, 91485, 32799, 106286, 97999, 65522, 32775] avg=65536.0000 max=106286 max_rank=4 min=32769 min_rank=1 imbalance_ratio=1.621796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[64653, 32769, 91485, 32799, 106286, 97999, 65522, 32775] avg=65536.0000 max=106286 max_rank=4 min=32769 min_rank=1 imbalance_ratio=1.621796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] avg=65536.0000 max=66192 max_rank=3 min=62758 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33373 max_load_reduction_pct=0.335188 moved_to_replicas=184945 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[94677, 34212, 65704, 98254, 32779, 33575, 65522, 99565] avg=65536.0000 max=99565 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.519241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[66192, 65486, 65806, 64635, 64236, 65549, 66192, 66192] avg=65536.0000 max=66192 max_rank=0 min=64236 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32076 max_load_reduction_pct=0.326413 moved_to_replicas=131497 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[94677, 34212, 65704, 98254, 32779, 33575, 65522, 99565] avg=65536.0000 max=99565 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.519241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[94677, 34212, 65704, 98254, 32779, 33575, 65522, 99565] avg=65536.0000 max=99565 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.519241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] avg=65536.0000 max=66193 max_rank=4 min=64606 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=40093 max_load_reduction_pct=0.377218 moved_to_replicas=194304 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] avg=65536.0000 max=66193 max_rank=4 min=64606 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=40093 max_load_reduction_pct=0.377218 moved_to_replicas=194304 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[94677, 34212, 65704, 98254, 32779, 33575, 65522, 99565] avg=65536.0000 max=99565 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.519241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] avg=65536.0000 max=66192 max_rank=3 min=62758 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33373 max_load_reduction_pct=0.335188 moved_to_replicas=184945 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] avg=65536.0000 max=66192 max_rank=3 min=62758 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33373 max_load_reduction_pct=0.335188 moved_to_replicas=184945 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] avg=65536.0000 max=66192 max_rank=3 min=62758 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33373 max_load_reduction_pct=0.335188 moved_to_replicas=184945 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[64653, 32769, 91485, 32799, 106286, 97999, 65522, 32775] avg=65536.0000 max=106286 max_rank=4 min=32769 min_rank=1 imbalance_ratio=1.621796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] avg=65536.0000 max=66192 max_rank=3 min=62758 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33373 max_load_reduction_pct=0.335188 moved_to_replicas=184945 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[94677, 34212, 65704, 98254, 32779, 33575, 65522, 99565] avg=65536.0000 max=99565 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.519241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[94677, 34212, 65704, 98254, 32779, 33575, 65522, 99565] avg=65536.0000 max=99565 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.519241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[64653, 65533, 66190, 64606, 66193, 66192, 65522, 65399] avg=65536.0000 max=66193 max_rank=4 min=64606 min_rank=3 imbalance_ratio=1.010025 max_load_reduction=40093 max_load_reduction_pct=0.377218 moved_to_replicas=194304 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] avg=65536.0000 max=66192 max_rank=3 min=62758 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33373 max_load_reduction_pct=0.335188 moved_to_replicas=184945 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] avg=65536.0000 max=66192 max_rank=3 min=62758 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33373 max_load_reduction_pct=0.335188 moved_to_replicas=184945 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[94677, 34212, 65704, 98254, 32779, 33575, 65522, 99565] avg=65536.0000 max=99565 max_rank=7 min=32779 min_rank=4 imbalance_ratio=1.519241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[66190, 62758, 65704, 66192, 65538, 66192, 65522, 66192] avg=65536.0000 max=66192 max_rank=3 min=62758 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33373 max_load_reduction_pct=0.335188 moved_to_replicas=184945 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=2093500 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507264 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=2093278 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507648 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=2090005 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=511104 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=2105100 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=495360 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=2093867 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507008 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=2098507 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=501504 peak_demand=66194 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=2105549 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=494976 peak_demand=66193 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=2097410 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=503040 peak_demand=66193 required_capacity=1.010025 + [2026-08-30 05:14:14] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 535.9 | throughput per GPU (TFLOP/s/GPU): 622.1 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.874065E+00 | loss scale: 1.0 | grad norm: 5.305 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 67328 | min safe capacity: 1.0273 | +[after training is done] datetime: 2026-08-30 05:14:14 +[rank0]:[W830 05:14:15.737385805 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 05:14:17.301733749 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 05:14:17.381264566 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 05:14:17.383192850 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 05:14:17.388945714 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 05:14:17.390031660 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 05:14:17.412326200 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 05:14:17.931805860 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:14:19.908000 495393 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:14:19.909000 495393 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 495485 closing signal SIGTERM +W0830 05:14:19.909000 495393 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 495486 closing signal SIGTERM +W0830 05:14:19.910000 495393 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 495487 closing signal SIGTERM +W0830 05:14:19.910000 495393 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 495488 closing signal SIGTERM +W0830 05:14:19.911000 495393 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 495489 closing signal SIGTERM +W0830 05:14:19.911000 495393 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 495490 closing signal SIGTERM +W0830 05:14:19.912000 495393 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 495491 closing signal SIGTERM +W0830 05:14:19.912000 495393 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 495492 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 495393 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..8cb7ab0fa8d45b0153dccad4d599756b17a4750c --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 2337 +1, 2345 +2, 2333 +3, 2347 +4, 2333 +5, 2337 +6, 2345 +7, 2353 +0, 19017 +1, 35145 +2, 35145 +3, 35145 +4, 19017 +5, 14867 +6, 35145 +7, 35145 +0, 42227 +1, 42213 +2, 42213 +3, 42213 +4, 42213 +5, 42213 +6, 42213 +7, 42213 +0, 43545 +1, 43529 +2, 43529 +3, 43529 +4, 43529 +5, 43529 +6, 43529 +7, 43529 +0, 43545 +1, 43529 +2, 43529 +3, 43529 +4, 43529 +5, 43529 +6, 43529 +7, 43529 +0, 44185 +1, 44169 +2, 44169 +3, 44169 +4, 44169 +5, 44169 +6, 44169 +7, 44169 +0, 46323 +1, 46307 +2, 46307 +3, 46307 +4, 46307 +5, 46307 +6, 46307 +7, 46307 +0, 75741 +1, 75725 +2, 75725 +3, 75725 +4, 75785 +5, 75725 +6, 75725 +7, 75725 +0, 76591 +1, 76575 +2, 76575 +3, 76575 +4, 76635 +5, 76575 +6, 76575 +7, 76575 +0, 76591 +1, 76575 +2, 76575 +3, 76575 +4, 76635 +5, 76575 +6, 76575 +7, 76575 +0, 82235 +1, 81325 +2, 81467 +3, 81487 +4, 82357 +5, 82251 +6, 81427 +7, 82337 +0, 89973 +1, 89957 +2, 89937 +3, 89947 +4, 89979 +5, 89967 +6, 90009 +7, 90037 +0, 90997 +1, 90991 +2, 90947 +3, 90989 +4, 90959 +5, 90989 +6, 90977 +7, 90991 +0, 114175 +1, 114267 +2, 114267 +3, 114265 +4, 114259 +5, 114267 +6, 114265 +7, 114267 +0, 114175 +1, 114267 +2, 114267 +3, 114265 +4, 114261 +5, 114267 +6, 114265 +7, 114267 +0, 114175 +1, 114267 +2, 114267 +3, 114265 +4, 114261 +5, 114267 +6, 114265 +7, 114267 +0, 113895 +1, 113987 +2, 113987 +3, 113985 +4, 113981 +5, 113987 +6, 113985 +7, 113987 +0, 109389 +1, 109481 +2, 109481 +3, 109479 +4, 109539 +5, 110127 +6, 109737 +7, 109801 +0, 107705 +1, 108059 +2, 107797 +3, 108057 +4, 107791 +5, 108059 +6, 107795 +7, 108059 +0, 4 +1, 4 +2, 4 +3, 107023 +4, 4 +5, 107025 +6, 107023 +7, 107025 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..3b7c11791905e2c265ccbc251bad6bc3be06583c --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G3 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=1792 +num_experts=64 +topk=16 +experts_per_rank=8 +expert_params=22020096 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=65536 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..aa51c9b1edde63e9ce2d170376e5aca44777d109 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log @@ -0,0 +1,9088 @@ +[05:15:39] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G3 / ultraep (slots/rank=8 policy=matched) + E=64 K=16 H'=1792 m=8 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:15:41.254000 502477 torch/distributed/run.py:766] +W0830 05:15:41.254000 502477 torch/distributed/run.py:766] ***************************************** +W0830 05:15:41.254000 502477 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:15:41.254000 502477 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1792 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 16 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank4]:[W830 05:15:59.950343210 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:15:59.950426126 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:15:59.950619244 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:15:59.952578797 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:15:59.952721756 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:15:59.952786174 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:15:59.954013316 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.032 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:15:59.102188112 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.814 seconds +time to initialize megatron (seconds): 7.262 +[after megatron is initialized] datetime: 2026-08-30 05:16:05 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6724653056 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42217472 elements, 42217472 padded size): + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight +Params for bucket 3 (42213376 elements, 42213376 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight +Params for bucket 4 (42213376 elements, 42213376 padded size): + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.pre_mlp_layernorm.weight +Params for bucket 5 (42213376 elements, 42213376 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 6 (42213376 elements, 42213376 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight +Params for bucket 7 (42213376 elements, 42213376 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 8 (42213376 elements, 42213376 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.weight +Params for bucket 9 (42213376 elements, 42213376 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 10 (42213376 elements, 42213376 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 11 (42213376 elements, 42213376 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight +Params for bucket 12 (42213376 elements, 42213376 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 13 (42213376 elements, 42213376 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_qkv.weight +Params for bucket 14 (42213376 elements, 42213376 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (42213376 elements, 42213376 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 16 (42213376 elements, 42213376 padded size): + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight +Params for bucket 17 (42213376 elements, 42213376 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 64 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 17 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 35 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 +Params for bucket 42 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 55 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 56 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 63 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:16:09 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:16:09 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3588.64, 3598.77) + train/valid/test-data-iterators-setup ..........: (73.40, 92.29) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:16:09 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131862 expected_overflow=0 exact_recv_per_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131862 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=81920 expected_recv=130215 expected_overflow=0 exact_recv_per_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131024 expected_overflow=0 exact_recv_per_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131024 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=81920 expected_recv=130534 expected_overflow=0 exact_recv_per_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131339 expected_overflow=0 exact_recv_per_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131173 expected_overflow=0 exact_recv_per_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=81920 expected_recv=130215 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131184 expected_overflow=0 exact_recv_per_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=81920 expected_recv=130534 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131339 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131056 expected_overflow=0 exact_recv_per_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131173 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131184 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131042 expected_overflow=0 exact_recv_per_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131042 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131056 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=81920 expected_recv=130654 expected_overflow=0 exact_recv_per_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=81920 expected_recv=130654 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131313 expected_overflow=0 exact_recv_per_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131289 expected_overflow=0 exact_recv_per_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131110 expected_overflow=0 exact_recv_per_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131313 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131290 expected_overflow=0 exact_recv_per_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131289 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=81920 expected_recv=130872 expected_overflow=0 exact_recv_per_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131110 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131290 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=81920 expected_recv=130872 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131054 expected_overflow=0 exact_recv_per_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131054 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131256 expected_overflow=0 exact_recv_per_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=81920 expected_recv=130793 expected_overflow=0 exact_recv_per_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131143 expected_overflow=0 exact_recv_per_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131129 expected_overflow=0 exact_recv_per_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131056 expected_overflow=0 exact_recv_per_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131256 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=81920 expected_recv=130793 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131129 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131143 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131056 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=81920 expected_recv=130966 expected_overflow=0 exact_recv_per_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=81920 expected_recv=130966 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131047 expected_overflow=0 exact_recv_per_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131047 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=81920 expected_recv=130705 expected_overflow=0 exact_recv_per_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=81920 expected_recv=131179 expected_overflow=0 exact_recv_per_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131626 expected_overflow=0 exact_recv_per_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131311 expected_overflow=0 exact_recv_per_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=81920 expected_recv=130421 expected_overflow=0 exact_recv_per_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=81920 expected_recv=130705 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=81920 expected_recv=131179 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131626 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=81920 expected_recv=130421 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131311 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131033 expected_overflow=0 exact_recv_per_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131411 expected_overflow=0 exact_recv_per_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131033 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131411 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131163 expected_overflow=0 exact_recv_per_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=81920 expected_recv=130875 expected_overflow=0 exact_recv_per_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131341 expected_overflow=0 exact_recv_per_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131220 expected_overflow=0 exact_recv_per_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131145 expected_overflow=0 exact_recv_per_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131163 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=81920 expected_recv=130875 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131341 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131220 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131145 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=81920 expected_recv=130751 expected_overflow=0 exact_recv_per_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=81920 expected_recv=130763 expected_overflow=0 exact_recv_per_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=81920 expected_recv=130751 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=81920 expected_recv=130763 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=81920 expected_recv=130880 expected_overflow=0 exact_recv_per_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=81920 expected_recv=131470 expected_overflow=0 exact_recv_per_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=81920 expected_recv=130970 expected_overflow=0 exact_recv_per_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=81920 expected_recv=131779 expected_overflow=0 exact_recv_per_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=81920 expected_recv=130880 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=81920 expected_recv=131470 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=81920 expected_recv=130612 expected_overflow=0 exact_recv_per_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=81920 expected_recv=130970 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=81920 expected_recv=131779 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=81920 expected_recv=130612 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131092 expected_overflow=0 exact_recv_per_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=81920 expected_recv=130602 expected_overflow=0 exact_recv_per_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131092 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=81920 expected_recv=130602 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=81920 expected_recv=130483 expected_overflow=0 exact_recv_per_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131257 expected_overflow=0 exact_recv_per_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131316 expected_overflow=0 exact_recv_per_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=81920 expected_recv=130483 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131257 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131248 expected_overflow=0 exact_recv_per_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131316 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=81920 expected_recv=130911 expected_overflow=0 exact_recv_per_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131248 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=81920 expected_recv=130591 expected_overflow=0 exact_recv_per_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=81920 expected_recv=130911 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=81920 expected_recv=130591 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=81920 expected_recv=130266 expected_overflow=0 exact_recv_per_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=81920 expected_recv=130266 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=81920 expected_recv=131781 expected_overflow=0 exact_recv_per_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=81920 expected_recv=131108 expected_overflow=0 exact_recv_per_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=81920 expected_recv=131781 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=81920 expected_recv=131429 expected_overflow=0 exact_recv_per_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=81920 expected_recv=131108 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=81920 expected_recv=131429 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=81920 expected_recv=130694 expected_overflow=0 exact_recv_per_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131144 expected_overflow=0 exact_recv_per_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=81920 expected_recv=130694 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=81920 expected_recv=130741 expected_overflow=0 exact_recv_per_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131144 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=81920 expected_recv=131388 expected_overflow=0 exact_recv_per_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=81920 expected_recv=130741 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131213 expected_overflow=0 exact_recv_per_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=81920 expected_recv=131388 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131300 expected_overflow=0 exact_recv_per_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131300 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=81920 expected_recv=130809 expected_overflow=0 exact_recv_per_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=81920 expected_recv=130789 expected_overflow=0 exact_recv_per_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=81920 expected_recv=130809 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=81920 expected_recv=130789 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=81920 expected_recv=131487 expected_overflow=0 exact_recv_per_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=81920 expected_recv=131435 expected_overflow=0 exact_recv_per_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=81920 expected_recv=131233 expected_overflow=0 exact_recv_per_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=81920 expected_recv=131213 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=81920 expected_recv=131487 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=81920 expected_recv=131435 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=81920 expected_recv=131421 expected_overflow=0 exact_recv_per_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=81920 expected_recv=131233 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=81920 expected_recv=131421 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131224 expected_overflow=0 exact_recv_per_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131224 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=81920 expected_recv=130844 expected_overflow=0 exact_recv_per_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=81920 expected_recv=130736 expected_overflow=0 exact_recv_per_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=81920 expected_recv=130844 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=81920 expected_recv=130736 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=81920 expected_recv=130700 expected_overflow=0 exact_recv_per_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131017 expected_overflow=0 exact_recv_per_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=81920 expected_recv=131596 expected_overflow=0 exact_recv_per_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=81920 expected_recv=130700 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131017 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=81920 expected_recv=131237 expected_overflow=0 exact_recv_per_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=81920 expected_recv=131596 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=81920 expected_recv=131237 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131058 expected_overflow=0 exact_recv_per_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=81920 expected_recv=130712 expected_overflow=0 exact_recv_per_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131117 expected_overflow=0 exact_recv_per_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131110 expected_overflow=0 exact_recv_per_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=81920 expected_recv=130712 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=81920 expected_recv=131058 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131117 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131110 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131335 expected_overflow=0 exact_recv_per_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131300 expected_overflow=0 exact_recv_per_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=81920 expected_recv=130811 expected_overflow=0 exact_recv_per_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131335 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131300 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131084 expected_overflow=0 exact_recv_per_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=81920 expected_recv=130811 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131084 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131340 expected_overflow=0 exact_recv_per_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131296 expected_overflow=0 exact_recv_per_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131340 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131211 expected_overflow=0 exact_recv_per_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131507 expected_overflow=0 exact_recv_per_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131296 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=81920 expected_recv=131211 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=81920 expected_recv=131507 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=81920 expected_recv=131091 expected_overflow=0 exact_recv_per_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=81920 expected_recv=130710 expected_overflow=0 exact_recv_per_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131062 expected_overflow=0 exact_recv_per_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=81920 expected_recv=130825 expected_overflow=0 exact_recv_per_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=81920 expected_recv=131091 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=81920 expected_recv=130710 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=81920 expected_recv=131062 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=81920 expected_recv=130825 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=81920 expected_recv=130943 expected_overflow=0 exact_recv_per_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=81920 expected_recv=130943 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=81920 expected_recv=131551 expected_overflow=0 exact_recv_per_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131309 expected_overflow=0 exact_recv_per_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=81920 expected_recv=130949 expected_overflow=0 exact_recv_per_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=81920 expected_recv=131551 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=81920 expected_recv=131309 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=81920 expected_recv=130949 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131368 expected_overflow=0 exact_recv_per_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=81920 expected_recv=131129 expected_overflow=0 exact_recv_per_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=81920 expected_recv=130828 expected_overflow=0 exact_recv_per_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=81920 expected_recv=131398 expected_overflow=0 exact_recv_per_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131368 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=81920 expected_recv=131129 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=81920 expected_recv=130828 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=81920 expected_recv=131398 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=81920 expected_recv=130902 expected_overflow=0 exact_recv_per_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=81920 expected_recv=130902 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131014 expected_overflow=0 exact_recv_per_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=81920 expected_recv=130970 expected_overflow=0 exact_recv_per_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131014 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131237 expected_overflow=0 exact_recv_per_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=81920 expected_recv=130847 expected_overflow=0 exact_recv_per_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=81920 expected_recv=130970 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=81920 expected_recv=130855 expected_overflow=0 exact_recv_per_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131359 expected_overflow=0 exact_recv_per_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=81920 expected_recv=130847 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=81920 expected_recv=131237 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=81920 expected_recv=130855 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131072 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=81920 expected_recv=131359 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131313 expected_overflow=0 exact_recv_per_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=81920 expected_recv=130745 expected_overflow=0 exact_recv_per_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131067 expected_overflow=0 exact_recv_per_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131313 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=81920 expected_recv=130902 expected_overflow=0 exact_recv_per_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=81920 expected_recv=130745 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=81920 expected_recv=130603 expected_overflow=0 exact_recv_per_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131067 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=81920 expected_recv=130902 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131058 expected_overflow=0 exact_recv_per_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131094 expected_overflow=0 exact_recv_per_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=81920 expected_recv=130603 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131058 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131094 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131197 expected_overflow=0 exact_recv_per_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131254 expected_overflow=0 exact_recv_per_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131197 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131254 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131090 expected_overflow=0 exact_recv_per_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131294 expected_overflow=0 exact_recv_per_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=81920 expected_recv=130787 expected_overflow=0 exact_recv_per_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=81920 expected_recv=131090 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131294 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=81920 expected_recv=130787 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131009 expected_overflow=0 exact_recv_per_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131009 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=81920 expected_recv=131051 expected_overflow=0 exact_recv_per_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131196 expected_overflow=0 exact_recv_per_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=81920 expected_recv=131051 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131196 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=81920 expected_recv=131250 expected_overflow=0 exact_recv_per_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=81920 expected_recv=131250 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=81920 expected_recv=131234 expected_overflow=0 exact_recv_per_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=81920 expected_recv=131234 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=81920 expected_recv=130551 expected_overflow=0 exact_recv_per_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=81920 expected_recv=130551 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=81920 expected_recv=130886 expected_overflow=0 exact_recv_per_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=81920 expected_recv=130886 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=81920 expected_recv=130891 expected_overflow=0 exact_recv_per_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=81920 expected_recv=130891 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=81920 expected_recv=130850 expected_overflow=0 exact_recv_per_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=81920 expected_recv=130850 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131282 expected_overflow=0 exact_recv_per_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=81920 expected_recv=131282 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131164 expected_overflow=0 exact_recv_per_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=81920 expected_recv=131164 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[65950, 65013, 65523, 65734, 65730, 65759, 65594, 64985] avg=65536.0000 max=65950 max_rank=0 min=64985 min_rank=7 imbalance_ratio=1.006317 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[65502, 65595, 65545, 65558, 65423, 65695, 65721, 65249] avg=65536.0000 max=65721 max_rank=6 min=65249 min_rank=7 imbalance_ratio=1.002823 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[65311, 65539, 65704, 65701, 65460, 65365, 65692, 65516] avg=65536.0000 max=65704 max_rank=2 min=65311 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[65648, 65258, 65804, 65821, 65596, 65109, 65524, 65528] avg=65536.0000 max=65821 max_rank=3 min=65109 min_rank=5 imbalance_ratio=1.004349 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[65472, 65404, 65684, 65274, 65445, 65842, 65235, 65932] avg=65536.0000 max=65932 max_rank=7 min=65235 min_rank=6 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[65573, 65920, 65240, 65808, 65247, 65526, 65596, 65378] avg=65536.0000 max=65920 max_rank=1 min=65240 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[65394, 65655, 65753, 65798, 65152, 65380, 65695, 65461] avg=65536.0000 max=65798 max_rank=3 min=65152 min_rank=4 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[65660, 65713, 65356, 65526, 65489, 65774, 65652, 65118] avg=65536.0000 max=65774 max_rank=5 min=65118 min_rank=7 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=2098254 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506880 peak_demand=65950 required_capacity=1.006317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[65395, 65286, 65756, 65750, 65714, 65270, 65802, 65315] avg=65536.0000 max=65802 max_rank=6 min=65270 min_rank=5 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=2095491 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509824 peak_demand=65932 required_capacity=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[65928, 65635, 65615, 65376, 65544, 65380, 65263, 65547] avg=65536.0000 max=65928 max_rank=0 min=65263 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=2097387 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507264 peak_demand=65962 required_capacity=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[65680, 65577, 65053, 65838, 65465, 65470, 65414, 65791] avg=65536.0000 max=65838 max_rank=3 min=65053 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=2096267 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508800 peak_demand=65932 required_capacity=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=2096685 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508928 peak_demand=65920 required_capacity=1.005859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[65547, 65836, 65253, 65475, 65547, 65746, 65156, 65728] avg=65536.0000 max=65836 max_rank=1 min=65156 min_rank=6 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=2098838 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506624 peak_demand=65971 required_capacity=1.006638 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=2097115 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508416 peak_demand=65802 required_capacity=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[65409, 65684, 65300, 65486, 65797, 65377, 65412, 65823] avg=65536.0000 max=65823 max_rank=7 min=65300 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[65408, 65347, 65298, 65530, 65932, 65545, 65318, 65910] avg=65536.0000 max=65932 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[65399, 65378, 65461, 65601, 65536, 65962, 65718, 65233] avg=65536.0000 max=65962 max_rank=5 min=65233 min_rank=7 imbalance_ratio=1.006500 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[65610, 65465, 65705, 65410, 65532, 65683, 65490, 65393] avg=65536.0000 max=65705 max_rank=2 min=65393 min_rank=7 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=2097179 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507392 peak_demand=65821 required_capacity=1.004349 +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB + [2026-08-30 05:16:36] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 26723.9 | throughput per GPU (TFLOP/s/GPU): 12.5 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.101823E+01 | loss scale: 1.0 | grad norm: 43.807 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66432 | min safe capacity: 1.0137 | +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58092.3076171875 | max allocated: 79556.37890625 | reserved: 82222.0 | max reserved: 82222.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131046 expected_overflow=0 exact_recv_per_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=81920 expected_recv=130869 expected_overflow=0 exact_recv_per_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=81920 expected_recv=130699 expected_overflow=0 exact_recv_per_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=81920 expected_recv=130833 expected_overflow=0 exact_recv_per_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131275 expected_overflow=0 exact_recv_per_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131369 expected_overflow=0 exact_recv_per_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131185 expected_overflow=0 exact_recv_per_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131046 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=81920 expected_recv=130869 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131300 expected_overflow=0 exact_recv_per_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=81920 expected_recv=130699 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=81920 expected_recv=130833 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131275 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131369 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131185 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=81920 expected_recv=131300 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131104 expected_overflow=0 exact_recv_per_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=81920 expected_recv=130959 expected_overflow=0 exact_recv_per_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131104 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=81920 expected_recv=130959 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131516 expected_overflow=0 exact_recv_per_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131032 expected_overflow=0 exact_recv_per_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=81920 expected_recv=130674 expected_overflow=0 exact_recv_per_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131436 expected_overflow=0 exact_recv_per_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131181 expected_overflow=0 exact_recv_per_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131516 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131032 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=81920 expected_recv=130674 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=81920 expected_recv=130674 expected_overflow=0 exact_recv_per_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131436 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=81920 expected_recv=131181 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=81920 expected_recv=130674 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=81920 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=81920 expected_recv=131297 expected_overflow=0 exact_recv_per_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=81920 expected_recv=131072 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=81920 expected_recv=131297 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=81920 expected_recv=130945 expected_overflow=0 exact_recv_per_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=81920 expected_recv=131148 expected_overflow=0 exact_recv_per_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=81920 expected_recv=130629 expected_overflow=0 exact_recv_per_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=81920 expected_recv=130771 expected_overflow=0 exact_recv_per_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=81920 expected_recv=131238 expected_overflow=0 exact_recv_per_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=81920 expected_recv=130945 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=81920 expected_recv=131148 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=81920 expected_recv=131476 expected_overflow=0 exact_recv_per_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=81920 expected_recv=130629 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=81920 expected_recv=130771 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=81920 expected_recv=131238 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=81920 expected_recv=131476 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131419 expected_overflow=0 exact_recv_per_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131419 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131024 expected_overflow=0 exact_recv_per_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=81920 expected_recv=130919 expected_overflow=0 exact_recv_per_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131024 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=81920 expected_recv=130742 expected_overflow=0 exact_recv_per_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=81920 expected_recv=130919 expected_overflow=0 exact_recv_per_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=81920 expected_recv=130919 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=81920 expected_recv=130968 expected_overflow=0 exact_recv_per_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=81920 expected_recv=130742 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131513 expected_overflow=0 exact_recv_per_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131072 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=81920 expected_recv=130919 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=81920 expected_recv=130968 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=81920 expected_recv=131513 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=81920 expected_recv=131025 expected_overflow=0 exact_recv_per_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=81920 expected_recv=131025 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=81920 expected_recv=130791 expected_overflow=0 exact_recv_per_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=81920 expected_recv=131220 expected_overflow=0 exact_recv_per_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=81920 expected_recv=130791 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=81920 expected_recv=131080 expected_overflow=0 exact_recv_per_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=81920 expected_recv=131462 expected_overflow=0 exact_recv_per_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=81920 expected_recv=131220 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=81920 expected_recv=130855 expected_overflow=0 exact_recv_per_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=81920 expected_recv=131080 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=81920 expected_recv=131462 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=81920 expected_recv=130959 expected_overflow=0 exact_recv_per_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=81920 expected_recv=130855 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130962 expected_overflow=0 exact_recv_per_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=81920 expected_recv=130959 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=81920 expected_recv=131184 expected_overflow=0 exact_recv_per_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130962 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=81920 expected_recv=131184 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=81920 expected_recv=131354 expected_overflow=0 exact_recv_per_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130927 expected_overflow=0 exact_recv_per_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=81920 expected_recv=131354 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130927 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130994 expected_overflow=0 exact_recv_per_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=81920 expected_recv=131238 expected_overflow=0 exact_recv_per_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130994 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130944 expected_overflow=0 exact_recv_per_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=81920 expected_recv=131238 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130880 expected_overflow=0 exact_recv_per_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130944 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=81920 expected_recv=130960 expected_overflow=0 exact_recv_per_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=81920 expected_recv=131277 expected_overflow=0 exact_recv_per_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=81920 expected_recv=130880 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=81920 expected_recv=130928 expected_overflow=0 exact_recv_per_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=81920 expected_recv=131277 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=81920 expected_recv=130960 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=81920 expected_recv=130928 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=81920 expected_recv=131042 expected_overflow=0 exact_recv_per_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=81920 expected_recv=130786 expected_overflow=0 exact_recv_per_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=81920 expected_recv=131042 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=81920 expected_recv=130786 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=81920 expected_recv=131280 expected_overflow=0 exact_recv_per_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=81920 expected_recv=131363 expected_overflow=0 exact_recv_per_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=81920 expected_recv=131280 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=81920 expected_recv=131363 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=81920 expected_recv=130755 expected_overflow=0 exact_recv_per_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=81920 expected_recv=131462 expected_overflow=0 exact_recv_per_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=81920 expected_recv=130755 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=81920 expected_recv=130926 expected_overflow=0 exact_recv_per_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=81920 expected_recv=131137 expected_overflow=0 exact_recv_per_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=81920 expected_recv=131462 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=81920 expected_recv=130926 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=81920 expected_recv=131137 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=81920 expected_recv=130722 expected_overflow=0 exact_recv_per_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=81920 expected_recv=131052 expected_overflow=0 exact_recv_per_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=81920 expected_recv=130722 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=81920 expected_recv=131052 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=81920 expected_recv=130952 expected_overflow=0 exact_recv_per_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=81920 expected_recv=131079 expected_overflow=0 exact_recv_per_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=81920 expected_recv=130952 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=81920 expected_recv=131079 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=81920 expected_recv=131323 expected_overflow=0 exact_recv_per_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131043 expected_overflow=0 exact_recv_per_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=81920 expected_recv=131323 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=81920 expected_recv=131385 expected_overflow=0 exact_recv_per_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131498 expected_overflow=0 exact_recv_per_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131043 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=81920 expected_recv=131385 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131032 expected_overflow=0 exact_recv_per_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131246 expected_overflow=0 exact_recv_per_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131498 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131032 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131246 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=81920 expected_recv=130962 expected_overflow=0 exact_recv_per_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=81920 expected_recv=130680 expected_overflow=0 exact_recv_per_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=81920 expected_recv=130962 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=81920 expected_recv=130680 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=81920 expected_recv=130751 expected_overflow=0 exact_recv_per_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=81920 expected_recv=130993 expected_overflow=0 exact_recv_per_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=81920 expected_recv=130751 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131364 expected_overflow=0 exact_recv_per_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=81920 expected_recv=130993 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131114 expected_overflow=0 exact_recv_per_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=81920 expected_recv=131364 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131104 expected_overflow=0 exact_recv_per_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131004 expected_overflow=0 exact_recv_per_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131114 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131104 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131004 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131020 expected_overflow=0 exact_recv_per_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131005 expected_overflow=0 exact_recv_per_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131020 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131005 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=81920 expected_recv=130867 expected_overflow=0 exact_recv_per_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=81920 expected_recv=130930 expected_overflow=0 exact_recv_per_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131469 expected_overflow=0 exact_recv_per_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=81920 expected_recv=130867 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=81920 expected_recv=130930 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=81920 expected_recv=131469 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=81920 expected_recv=131094 expected_overflow=0 exact_recv_per_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=81920 expected_recv=131430 expected_overflow=0 exact_recv_per_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=81920 expected_recv=130963 expected_overflow=0 exact_recv_per_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=81920 expected_recv=131094 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=81920 expected_recv=131430 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=81920 expected_recv=130963 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=81920 expected_recv=130781 expected_overflow=0 exact_recv_per_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=81920 expected_recv=130873 expected_overflow=0 exact_recv_per_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=81920 expected_recv=130781 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=81920 expected_recv=130873 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=81920 expected_recv=131274 expected_overflow=0 exact_recv_per_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=81920 expected_recv=131144 expected_overflow=0 exact_recv_per_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=81920 expected_recv=131274 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=81920 expected_recv=131361 expected_overflow=0 exact_recv_per_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=81920 expected_recv=131144 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=81920 expected_recv=130732 expected_overflow=0 exact_recv_per_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=81920 expected_recv=131361 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=81920 expected_recv=130990 expected_overflow=0 exact_recv_per_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=81920 expected_recv=131139 expected_overflow=0 exact_recv_per_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=81920 expected_recv=130732 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=81920 expected_recv=130990 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=81920 expected_recv=131139 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=81920 expected_recv=131133 expected_overflow=0 exact_recv_per_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=81920 expected_recv=130790 expected_overflow=0 exact_recv_per_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=81920 expected_recv=131133 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=81920 expected_recv=130790 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=81920 expected_recv=130880 expected_overflow=0 exact_recv_per_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=81920 expected_recv=131630 expected_overflow=0 exact_recv_per_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=81920 expected_recv=130888 expected_overflow=0 exact_recv_per_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=81920 expected_recv=130880 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=81920 expected_recv=130935 expected_overflow=0 exact_recv_per_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=81920 expected_recv=131630 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=81920 expected_recv=131172 expected_overflow=0 exact_recv_per_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=81920 expected_recv=130888 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=81920 expected_recv=130935 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=81920 expected_recv=131317 expected_overflow=0 exact_recv_per_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=81920 expected_recv=131172 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=81920 expected_recv=131317 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=81920 expected_recv=130982 expected_overflow=0 exact_recv_per_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=81920 expected_recv=130982 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=81920 expected_recv=131160 expected_overflow=0 exact_recv_per_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=81920 expected_recv=131160 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130712 expected_overflow=0 exact_recv_per_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=81920 expected_recv=130978 expected_overflow=0 exact_recv_per_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=81920 expected_recv=131152 expected_overflow=0 exact_recv_per_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130712 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=81920 expected_recv=131278 expected_overflow=0 exact_recv_per_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=81920 expected_recv=130978 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=81920 expected_recv=131152 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=81920 expected_recv=131520 expected_overflow=0 exact_recv_per_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=81920 expected_recv=131278 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130839 expected_overflow=0 exact_recv_per_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=81920 expected_recv=131520 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130839 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130748 expected_overflow=0 exact_recv_per_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130748 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=81920 expected_recv=131461 expected_overflow=0 exact_recv_per_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131272 expected_overflow=0 exact_recv_per_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=81920 expected_recv=131461 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131272 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130896 expected_overflow=0 exact_recv_per_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=81920 expected_recv=130933 expected_overflow=0 exact_recv_per_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=81920 expected_recv=131122 expected_overflow=0 exact_recv_per_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131222 expected_overflow=0 exact_recv_per_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=81920 expected_recv=130612 expected_overflow=0 exact_recv_per_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=81920 expected_recv=130933 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=81920 expected_recv=130896 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=81920 expected_recv=131122 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131222 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=81920 expected_recv=130612 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131117 expected_overflow=0 exact_recv_per_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131117 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131253 expected_overflow=0 exact_recv_per_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131515 expected_overflow=0 exact_recv_per_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131253 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131515 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131486 expected_overflow=0 exact_recv_per_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=81920 expected_recv=130165 expected_overflow=0 exact_recv_per_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131131 expected_overflow=0 exact_recv_per_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131316 expected_overflow=0 exact_recv_per_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131486 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=81920 expected_recv=130165 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131131 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131316 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=81920 expected_recv=130833 expected_overflow=0 exact_recv_per_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131036 expected_overflow=0 exact_recv_per_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=81920 expected_recv=130833 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=81920 expected_recv=131036 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131086 expected_overflow=0 exact_recv_per_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131086 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131049 expected_overflow=0 exact_recv_per_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131049 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131126 expected_overflow=0 exact_recv_per_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=81920 expected_recv=131126 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[65393, 65257, 65195, 65483, 65889, 65779, 65538, 65754] avg=65536.0000 max=65889 max_rank=4 min=65195 min_rank=2 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[65569, 65571, 65364, 65445, 65502, 65665, 65427, 65745] avg=65536.0000 max=65745 max_rank=7 min=65364 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[65539, 65737, 65278, 65806, 65482, 65702, 65428, 65316] avg=65536.0000 max=65806 max_rank=3 min=65278 min_rank=2 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[65832, 65449, 65690, 65658, 65307, 65676, 65308, 65368] avg=65536.0000 max=65832 max_rank=0 min=65307 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[65452, 65433, 65357, 65758, 65658, 65608, 65882, 65140] avg=65536.0000 max=65882 max_rank=6 min=65140 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[65771, 65595, 65553, 65634, 65432, 65501, 65465, 65337] avg=65536.0000 max=65771 max_rank=0 min=65337 min_rank=7 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[65576, 65399, 65134, 65718, 65569, 65677, 65458, 65757] avg=65536.0000 max=65757 max_rank=7 min=65134 min_rank=2 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[65370, 65529, 65835, 65610, 65344, 65435, 65643, 65522] avg=65536.0000 max=65835 max_rank=2 min=65344 min_rank=4 imbalance_ratio=1.004562 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[65583, 65688, 65282, 65851, 65768, 65324, 65273, 65519] avg=65536.0000 max=65851 max_rank=3 min=65273 min_rank=6 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[65475, 65866, 65198, 65656, 65550, 65635, 65575, 65333] avg=65536.0000 max=65866 max_rank=1 min=65198 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[65597, 65656, 65508, 65707, 65419, 65559, 65445, 65397] avg=65536.0000 max=65707 max_rank=3 min=65397 min_rank=7 imbalance_ratio=1.002609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[65536, 65560, 65518, 65318, 65716, 65589, 65446, 65605] avg=65536.0000 max=65716 max_rank=4 min=65318 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[65581, 65591, 65507, 65353, 65522, 65381, 65800, 65553] avg=65536.0000 max=65800 max_rank=6 min=65353 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[65268, 65698, 65622, 65596, 65296, 65934, 65890, 64984] avg=65536.0000 max=65934 max_rank=5 min=64984 min_rank=7 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[65563, 65312, 65344, 65919, 65281, 65525, 65563, 65781] avg=65536.0000 max=65919 max_rank=3 min=65281 min_rank=4 imbalance_ratio=1.005844 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[65779, 64902, 65660, 65444, 65689, 65837, 65526, 65451] avg=65536.0000 max=65837 max_rank=5 min=64902 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=2097564 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507264 peak_demand=65934 required_capacity=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=2097213 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508672 peak_demand=65832 required_capacity=1.004517 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=2096185 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508288 peak_demand=65781 required_capacity=1.003738 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=2097390 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507136 peak_demand=65910 required_capacity=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=2097002 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508672 peak_demand=66014 required_capacity=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=2095315 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509184 peak_demand=66112 required_capacity=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=2099807 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=505600 peak_demand=65919 required_capacity=1.005844 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=2096740 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508928 peak_demand=65890 required_capacity=1.005402 + [2026-08-30 05:16:37] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1050.0 | throughput per GPU (TFLOP/s/GPU): 317.5 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101411E+01 | loss scale: 1.0 | grad norm: 45.665 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66432 | min safe capacity: 1.0137 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=81920 expected_recv=130955 expected_overflow=0 exact_recv_per_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=81920 expected_recv=130385 expected_overflow=0 exact_recv_per_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=81920 expected_recv=131232 expected_overflow=0 exact_recv_per_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=81920 expected_recv=131693 expected_overflow=0 exact_recv_per_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=81920 expected_recv=131048 expected_overflow=0 exact_recv_per_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=81920 expected_recv=130999 expected_overflow=0 exact_recv_per_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=81920 expected_recv=131067 expected_overflow=0 exact_recv_per_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=81920 expected_recv=130955 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=81920 expected_recv=130385 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=81920 expected_recv=131232 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=81920 expected_recv=131693 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=81920 expected_recv=131197 expected_overflow=0 exact_recv_per_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=81920 expected_recv=131048 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=81920 expected_recv=130999 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=81920 expected_recv=131067 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=81920 expected_recv=131197 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=81920 expected_recv=130867 expected_overflow=0 exact_recv_per_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=81920 expected_recv=131050 expected_overflow=0 exact_recv_per_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=81920 expected_recv=131294 expected_overflow=0 exact_recv_per_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=81920 expected_recv=130968 expected_overflow=0 exact_recv_per_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=81920 expected_recv=130909 expected_overflow=0 exact_recv_per_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=81920 expected_recv=131681 expected_overflow=0 exact_recv_per_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=81920 expected_recv=130867 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=81920 expected_recv=131050 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=81920 expected_recv=131294 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=81920 expected_recv=130909 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=81920 expected_recv=130968 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=81920 expected_recv=130941 expected_overflow=0 exact_recv_per_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=81920 expected_recv=131681 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=81920 expected_recv=130866 expected_overflow=0 exact_recv_per_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=81920 expected_recv=130941 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=81920 expected_recv=130866 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=81920 expected_recv=130977 expected_overflow=0 exact_recv_per_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=81920 expected_recv=130622 expected_overflow=0 exact_recv_per_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=81920 expected_recv=131248 expected_overflow=0 exact_recv_per_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=81920 expected_recv=131080 expected_overflow=0 exact_recv_per_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=81920 expected_recv=130977 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=81920 expected_recv=130622 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=81920 expected_recv=131248 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=81920 expected_recv=131080 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=81920 expected_recv=131068 expected_overflow=0 exact_recv_per_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=81920 expected_recv=130752 expected_overflow=0 exact_recv_per_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=81920 expected_recv=131670 expected_overflow=0 exact_recv_per_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=81920 expected_recv=131159 expected_overflow=0 exact_recv_per_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=81920 expected_recv=131068 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=81920 expected_recv=130752 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=81920 expected_recv=131670 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=81920 expected_recv=131159 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130985 expected_overflow=0 exact_recv_per_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130985 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=81920 expected_recv=131617 expected_overflow=0 exact_recv_per_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=81920 expected_recv=131112 expected_overflow=0 exact_recv_per_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130773 expected_overflow=0 exact_recv_per_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=81920 expected_recv=131617 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=81920 expected_recv=131112 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130773 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=81920 expected_recv=131206 expected_overflow=0 exact_recv_per_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=81920 expected_recv=131142 expected_overflow=0 exact_recv_per_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130899 expected_overflow=0 exact_recv_per_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130842 expected_overflow=0 exact_recv_per_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=81920 expected_recv=131206 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=81920 expected_recv=131142 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130899 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=81920 expected_recv=130842 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=81920 expected_recv=130661 expected_overflow=0 exact_recv_per_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=81920 expected_recv=130661 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131366 expected_overflow=0 exact_recv_per_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=81920 expected_recv=130781 expected_overflow=0 exact_recv_per_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131232 expected_overflow=0 exact_recv_per_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131366 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131087 expected_overflow=0 exact_recv_per_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=81920 expected_recv=130781 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131057 expected_overflow=0 exact_recv_per_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131232 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=81920 expected_recv=130719 expected_overflow=0 exact_recv_per_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131673 expected_overflow=0 exact_recv_per_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131087 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131057 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=81920 expected_recv=130719 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=81920 expected_recv=131673 actual_valid_recv=133120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131508 expected_overflow=0 exact_recv_per_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=133120 slack_slots=30720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131508 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131115 expected_overflow=0 exact_recv_per_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131041 expected_overflow=0 exact_recv_per_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131115 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131371 expected_overflow=0 exact_recv_per_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131041 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131205 expected_overflow=0 exact_recv_per_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=81920 expected_recv=130881 expected_overflow=0 exact_recv_per_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131371 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=81920 expected_recv=130784 expected_overflow=0 exact_recv_per_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=81920 expected_recv=130671 expected_overflow=0 exact_recv_per_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=81920 expected_recv=131205 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=81920 expected_recv=130881 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=81920 expected_recv=130824 expected_overflow=0 exact_recv_per_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=81920 expected_recv=130784 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=81920 expected_recv=130671 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=81920 expected_recv=130824 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=81920 expected_recv=130553 expected_overflow=0 exact_recv_per_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=81920 expected_recv=131280 expected_overflow=0 exact_recv_per_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=81920 expected_recv=130553 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=81920 expected_recv=130919 expected_overflow=0 exact_recv_per_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=81920 expected_recv=131280 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=81920 expected_recv=130919 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=81920 expected_recv=131521 expected_overflow=0 exact_recv_per_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=81920 expected_recv=130824 expected_overflow=0 exact_recv_per_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=81920 expected_recv=131415 expected_overflow=0 exact_recv_per_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=81920 expected_recv=131521 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=81920 expected_recv=131240 expected_overflow=0 exact_recv_per_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=81920 expected_recv=130824 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131015 expected_overflow=0 exact_recv_per_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=81920 expected_recv=131240 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131207 expected_overflow=0 exact_recv_per_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=81920 expected_recv=131415 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131015 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=81920 expected_recv=130699 expected_overflow=0 exact_recv_per_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131207 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=81920 expected_recv=130699 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131196 expected_overflow=0 exact_recv_per_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=81920 expected_recv=130878 expected_overflow=0 exact_recv_per_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131196 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131381 expected_overflow=0 exact_recv_per_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=81920 expected_recv=130878 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131502 expected_overflow=0 exact_recv_per_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131381 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=81920 expected_recv=131397 expected_overflow=0 exact_recv_per_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=81920 expected_recv=130698 expected_overflow=0 exact_recv_per_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=81920 expected_recv=130289 expected_overflow=0 exact_recv_per_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=81920 expected_recv=131502 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=81920 expected_recv=131397 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=81920 expected_recv=130698 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=81920 expected_recv=130289 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=81920 expected_recv=130981 expected_overflow=0 exact_recv_per_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=81920 expected_recv=130981 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=81920 expected_recv=131342 expected_overflow=0 exact_recv_per_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=81920 expected_recv=131745 expected_overflow=0 exact_recv_per_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=81920 expected_recv=131342 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=81920 expected_recv=130845 expected_overflow=0 exact_recv_per_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=81920 expected_recv=131745 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=81920 expected_recv=130845 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=81920 expected_recv=130927 expected_overflow=0 exact_recv_per_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=81920 expected_recv=131050 expected_overflow=0 exact_recv_per_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131426 expected_overflow=0 exact_recv_per_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131098 expected_overflow=0 exact_recv_per_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=81920 expected_recv=130927 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=81920 expected_recv=131050 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131426 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131098 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=81920 expected_recv=130911 expected_overflow=0 exact_recv_per_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131377 expected_overflow=0 exact_recv_per_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=81920 expected_recv=130611 expected_overflow=0 exact_recv_per_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=81920 expected_recv=130830 expected_overflow=0 exact_recv_per_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=81920 expected_recv=130911 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131377 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=81920 expected_recv=130611 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=81920 expected_recv=130830 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131517 expected_overflow=0 exact_recv_per_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=81920 expected_recv=130806 expected_overflow=0 exact_recv_per_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=81920 expected_recv=130903 expected_overflow=0 exact_recv_per_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=81920 expected_recv=130589 expected_overflow=0 exact_recv_per_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=81920 expected_recv=131517 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=81920 expected_recv=130806 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=81920 expected_recv=130903 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=81920 expected_recv=130589 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131333 expected_overflow=0 exact_recv_per_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131018 expected_overflow=0 exact_recv_per_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131211 expected_overflow=0 exact_recv_per_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131195 expected_overflow=0 exact_recv_per_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131333 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131018 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131211 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131195 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131227 expected_overflow=0 exact_recv_per_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131057 expected_overflow=0 exact_recv_per_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131100 expected_overflow=0 exact_recv_per_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=81920 expected_recv=130843 expected_overflow=0 exact_recv_per_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131227 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131057 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=81920 expected_recv=131100 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=81920 expected_recv=130843 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131229 expected_overflow=0 exact_recv_per_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=81920 expected_recv=130833 expected_overflow=0 exact_recv_per_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131158 expected_overflow=0 exact_recv_per_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131217 expected_overflow=0 exact_recv_per_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131229 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=81920 expected_recv=130833 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131158 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131217 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131105 expected_overflow=0 exact_recv_per_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=81920 expected_recv=130873 expected_overflow=0 exact_recv_per_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131134 expected_overflow=0 exact_recv_per_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131105 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=81920 expected_recv=131058 expected_overflow=0 exact_recv_per_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=81920 expected_recv=130873 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=81920 expected_recv=131134 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=81920 expected_recv=131058 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=81920 expected_recv=131155 expected_overflow=0 exact_recv_per_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=81920 expected_recv=130742 expected_overflow=0 exact_recv_per_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=81920 expected_recv=131126 expected_overflow=0 exact_recv_per_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=81920 expected_recv=131754 expected_overflow=0 exact_recv_per_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=81920 expected_recv=131155 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=81920 expected_recv=130742 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=81920 expected_recv=131126 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=81920 expected_recv=131754 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=81920 expected_recv=130801 expected_overflow=0 exact_recv_per_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=81920 expected_recv=131459 expected_overflow=0 exact_recv_per_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=81920 expected_recv=131067 expected_overflow=0 exact_recv_per_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=81920 expected_recv=130801 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=81920 expected_recv=131459 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=81920 expected_recv=131148 expected_overflow=0 exact_recv_per_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=81920 expected_recv=131067 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=81920 expected_recv=131148 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=81920 expected_recv=131517 expected_overflow=0 exact_recv_per_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=81920 expected_recv=130819 expected_overflow=0 exact_recv_per_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=81920 expected_recv=131318 expected_overflow=0 exact_recv_per_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=81920 expected_recv=130877 expected_overflow=0 exact_recv_per_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=81920 expected_recv=131517 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=81920 expected_recv=130819 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=81920 expected_recv=131318 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=81920 expected_recv=130877 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=81920 expected_recv=130381 expected_overflow=0 exact_recv_per_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=81920 expected_recv=130848 expected_overflow=0 exact_recv_per_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=81920 expected_recv=131057 expected_overflow=0 exact_recv_per_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=81920 expected_recv=130381 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=81920 expected_recv=130848 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=81920 expected_recv=131110 expected_overflow=0 exact_recv_per_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=81920 expected_recv=131057 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=81920 expected_recv=131110 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=81920 expected_recv=130989 expected_overflow=0 exact_recv_per_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=81920 expected_recv=130942 expected_overflow=0 exact_recv_per_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=81920 expected_recv=131091 expected_overflow=0 exact_recv_per_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=81920 expected_recv=130922 expected_overflow=0 exact_recv_per_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=81920 expected_recv=130989 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=81920 expected_recv=130942 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=81920 expected_recv=131091 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=81920 expected_recv=130922 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=81920 expected_recv=131016 expected_overflow=0 exact_recv_per_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=81920 expected_recv=130713 expected_overflow=0 exact_recv_per_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=81920 expected_recv=131658 expected_overflow=0 exact_recv_per_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=81920 expected_recv=131016 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=81920 expected_recv=130713 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=81920 expected_recv=130864 expected_overflow=0 exact_recv_per_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=81920 expected_recv=131658 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=81920 expected_recv=130864 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=81920 expected_recv=131401 expected_overflow=0 exact_recv_per_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=81920 expected_recv=130729 expected_overflow=0 exact_recv_per_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=81920 expected_recv=131302 expected_overflow=0 exact_recv_per_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=81920 expected_recv=131296 expected_overflow=0 exact_recv_per_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=81920 expected_recv=131401 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=81920 expected_recv=130729 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=81920 expected_recv=131302 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=81920 expected_recv=131296 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=81920 expected_recv=131064 expected_overflow=0 exact_recv_per_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=81920 expected_recv=131207 expected_overflow=0 exact_recv_per_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=81920 expected_recv=131064 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=81920 expected_recv=131207 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[65391, 65838, 65716, 65695, 65426, 65070, 65737, 65415] avg=65536.0000 max=65838 max_rank=1 min=65070 min_rank=5 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[65527, 65594, 65705, 65268, 65283, 65670, 65418, 65823] avg=65536.0000 max=65823 max_rank=7 min=65268 min_rank=3 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[65523, 65335, 65260, 65598, 65443, 65563, 65866, 65700] avg=65536.0000 max=65866 max_rank=6 min=65260 min_rank=2 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[65490, 65438, 65655, 65376, 65292, 65600, 65646, 65791] avg=65536.0000 max=65791 max_rank=7 min=65292 min_rank=4 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[65217, 65654, 65504, 65547, 65539, 65581, 65621, 65625] avg=65536.0000 max=65654 max_rank=1 min=65217 min_rank=0 imbalance_ratio=1.001801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[65711, 65442, 65909, 65300, 65397, 65696, 65269, 65564] avg=65536.0000 max=65909 max_rank=2 min=65269 min_rank=6 imbalance_ratio=1.005692 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[65520, 65580, 65423, 65701, 65608, 65260, 65641, 65555] avg=65536.0000 max=65701 max_rank=3 min=65260 min_rank=5 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[65294, 65891, 65720, 65136, 65224, 65503, 65680, 65840] avg=65536.0000 max=65891 max_rank=1 min=65136 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[65852, 65432, 65702, 65235, 65514, 65480, 65383, 65690] avg=65536.0000 max=65852 max_rank=0 min=65235 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[65972, 65381, 65265, 65510, 65397, 65584, 65889, 65290] avg=65536.0000 max=65972 max_rank=0 min=65265 min_rank=2 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[65549, 65527, 65805, 65523, 65488, 65017, 65546, 65833] avg=65536.0000 max=65833 max_rank=7 min=65017 min_rank=5 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[65385, 65689, 65658, 65644, 65330, 65420, 65446, 65716] avg=65536.0000 max=65716 max_rank=7 min=65330 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[65390, 66133, 65889, 65441, 65388, 65337, 65329, 65381] avg=65536.0000 max=66133 max_rank=1 min=65329 min_rank=6 imbalance_ratio=1.009109 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[65710, 65499, 65534, 65336, 65707, 65675, 65147, 65680] avg=65536.0000 max=65710 max_rank=0 min=65147 min_rank=6 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[65674, 65735, 65490, 65394, 65859, 65313, 65281, 65542] avg=65536.0000 max=65859 max_rank=4 min=65281 min_rank=6 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[65255, 65721, 65496, 65111, 65750, 65720, 65517, 65718] avg=65536.0000 max=65750 max_rank=4 min=65111 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=2096468 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508672 peak_demand=65972 required_capacity=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=2095540 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=510976 peak_demand=66017 required_capacity=1.007339 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=2099207 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506368 peak_demand=66055 required_capacity=1.007919 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=2097173 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508416 peak_demand=66133 required_capacity=1.009109 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=2099443 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=505728 peak_demand=66112 required_capacity=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=2097701 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507264 peak_demand=66052 required_capacity=1.007874 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=2094925 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509440 peak_demand=65746 required_capacity=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=2096759 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508032 peak_demand=65859 required_capacity=1.004929 + [2026-08-30 05:16:37] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 476.8 | throughput per GPU (TFLOP/s/GPU): 699.2 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.057920E+01 | loss scale: 1.0 | grad norm: 217.562 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66560 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=81920 expected_recv=130799 expected_overflow=0 exact_recv_per_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=81920 expected_recv=130891 expected_overflow=0 exact_recv_per_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=81920 expected_recv=131506 expected_overflow=0 exact_recv_per_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=81920 expected_recv=131075 expected_overflow=0 exact_recv_per_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=81920 expected_recv=130860 expected_overflow=0 exact_recv_per_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=81920 expected_recv=131485 expected_overflow=0 exact_recv_per_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=81920 expected_recv=130873 expected_overflow=0 exact_recv_per_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=81920 expected_recv=130799 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=81920 expected_recv=130891 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=81920 expected_recv=131506 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=81920 expected_recv=131075 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=81920 expected_recv=130860 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=81920 expected_recv=131087 expected_overflow=0 exact_recv_per_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=81920 expected_recv=131485 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=81920 expected_recv=130873 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=81920 expected_recv=131087 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=81920 expected_recv=131308 expected_overflow=0 exact_recv_per_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=81920 expected_recv=131308 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130877 expected_overflow=0 exact_recv_per_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130875 expected_overflow=0 exact_recv_per_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=81920 expected_recv=131134 expected_overflow=0 exact_recv_per_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130973 expected_overflow=0 exact_recv_per_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130877 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130711 expected_overflow=0 exact_recv_per_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130875 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=81920 expected_recv=131134 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130973 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=81920 expected_recv=131410 expected_overflow=0 exact_recv_per_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=81920 expected_recv=130711 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=81920 expected_recv=131288 expected_overflow=0 exact_recv_per_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=81920 expected_recv=131410 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131646 expected_overflow=0 exact_recv_per_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=81920 expected_recv=131288 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131646 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=81920 expected_recv=130745 expected_overflow=0 exact_recv_per_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=81920 expected_recv=130745 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131626 expected_overflow=0 exact_recv_per_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=81920 expected_recv=130485 expected_overflow=0 exact_recv_per_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131039 expected_overflow=0 exact_recv_per_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131086 expected_overflow=0 exact_recv_per_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=81920 expected_recv=130897 expected_overflow=0 exact_recv_per_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131626 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=81920 expected_recv=130485 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131039 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131086 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131097 expected_overflow=0 exact_recv_per_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131052 expected_overflow=0 exact_recv_per_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=81920 expected_recv=130897 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131097 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=81920 expected_recv=131052 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=81920 expected_recv=130734 expected_overflow=0 exact_recv_per_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=81920 expected_recv=130734 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=81920 expected_recv=130487 expected_overflow=0 exact_recv_per_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131513 expected_overflow=0 exact_recv_per_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131561 expected_overflow=0 exact_recv_per_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=81920 expected_recv=130675 expected_overflow=0 exact_recv_per_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=81920 expected_recv=130487 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131513 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131561 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=81920 expected_recv=130675 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131409 expected_overflow=0 exact_recv_per_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131019 expected_overflow=0 exact_recv_per_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131100 expected_overflow=0 exact_recv_per_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131409 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131013 expected_overflow=0 exact_recv_per_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131019 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=81920 expected_recv=131100 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131013 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=81920 expected_recv=130990 expected_overflow=0 exact_recv_per_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131215 expected_overflow=0 exact_recv_per_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=81920 expected_recv=130513 expected_overflow=0 exact_recv_per_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=81920 expected_recv=130990 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=81920 expected_recv=130885 expected_overflow=0 exact_recv_per_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131215 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=81920 expected_recv=130513 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=81920 expected_recv=130885 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131205 expected_overflow=0 exact_recv_per_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131107 expected_overflow=0 exact_recv_per_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131736 expected_overflow=0 exact_recv_per_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131205 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=81920 expected_recv=130609 expected_overflow=0 exact_recv_per_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131107 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=81920 expected_recv=131736 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=81920 expected_recv=130609 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131551 expected_overflow=0 exact_recv_per_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=81920 expected_recv=130895 expected_overflow=0 exact_recv_per_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131528 expected_overflow=0 exact_recv_per_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131551 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=81920 expected_recv=130905 expected_overflow=0 exact_recv_per_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=81920 expected_recv=130895 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131112 expected_overflow=0 exact_recv_per_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=81920 expected_recv=130905 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131528 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131547 expected_overflow=0 exact_recv_per_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=81920 expected_recv=130869 expected_overflow=0 exact_recv_per_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=81920 expected_recv=131112 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=81920 expected_recv=130619 expected_overflow=0 exact_recv_per_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131547 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=81920 expected_recv=130869 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=81920 expected_recv=130619 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=81920 expected_recv=130782 expected_overflow=0 exact_recv_per_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131028 expected_overflow=0 exact_recv_per_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=81920 expected_recv=130953 expected_overflow=0 exact_recv_per_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131329 expected_overflow=0 exact_recv_per_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=81920 expected_recv=130782 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131028 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=81920 expected_recv=130953 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=81920 expected_recv=130949 expected_overflow=0 exact_recv_per_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131329 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131369 expected_overflow=0 exact_recv_per_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=81920 expected_recv=130768 expected_overflow=0 exact_recv_per_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=81920 expected_recv=131173 expected_overflow=0 exact_recv_per_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=81920 expected_recv=130949 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=81920 expected_recv=131369 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=81920 expected_recv=130768 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=81920 expected_recv=131173 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=81920 expected_recv=130925 expected_overflow=0 exact_recv_per_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=81920 expected_recv=131111 expected_overflow=0 exact_recv_per_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=81920 expected_recv=131164 expected_overflow=0 exact_recv_per_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=81920 expected_recv=130925 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=81920 expected_recv=131111 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=81920 expected_recv=131164 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=81920 expected_recv=130894 expected_overflow=0 exact_recv_per_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=81920 expected_recv=131571 expected_overflow=0 exact_recv_per_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131034 expected_overflow=0 exact_recv_per_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=81920 expected_recv=130806 expected_overflow=0 exact_recv_per_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=81920 expected_recv=130970 expected_overflow=0 exact_recv_per_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=81920 expected_recv=130894 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=81920 expected_recv=131571 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131034 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=81920 expected_recv=130806 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=81920 expected_recv=130970 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131229 expected_overflow=0 exact_recv_per_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131376 expected_overflow=0 exact_recv_per_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=81920 expected_recv=130888 expected_overflow=0 exact_recv_per_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131229 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131376 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=81920 expected_recv=130888 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131369 expected_overflow=0 exact_recv_per_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131378 expected_overflow=0 exact_recv_per_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131186 expected_overflow=0 exact_recv_per_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=81920 expected_recv=130727 expected_overflow=0 exact_recv_per_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=81920 expected_recv=130496 expected_overflow=0 exact_recv_per_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131369 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=81920 expected_recv=131378 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131186 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=81920 expected_recv=130727 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=81920 expected_recv=130496 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=81920 expected_recv=130412 expected_overflow=0 exact_recv_per_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131519 expected_overflow=0 exact_recv_per_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=81920 expected_recv=130412 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131308 expected_overflow=0 exact_recv_per_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131519 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131308 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=81920 expected_recv=130965 expected_overflow=0 exact_recv_per_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=81920 expected_recv=130264 expected_overflow=0 exact_recv_per_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131168 expected_overflow=0 exact_recv_per_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131291 expected_overflow=0 exact_recv_per_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131484 expected_overflow=0 exact_recv_per_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=81920 expected_recv=130965 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=81920 expected_recv=130264 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131168 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=81920 expected_recv=131291 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131484 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131599 expected_overflow=0 exact_recv_per_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131197 expected_overflow=0 exact_recv_per_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131131 expected_overflow=0 exact_recv_per_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131599 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131197 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131131 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=81920 expected_recv=130629 expected_overflow=0 exact_recv_per_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131412 expected_overflow=0 exact_recv_per_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131168 expected_overflow=0 exact_recv_per_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131539 expected_overflow=0 exact_recv_per_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131104 expected_overflow=0 exact_recv_per_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=81920 expected_recv=130629 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131412 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131168 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131539 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=81920 expected_recv=131104 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=81920 expected_recv=130873 expected_overflow=0 exact_recv_per_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=81920 expected_recv=130705 expected_overflow=0 exact_recv_per_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131172 expected_overflow=0 exact_recv_per_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=81920 expected_recv=130873 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=81920 expected_recv=130705 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131172 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130953 expected_overflow=0 exact_recv_per_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131168 expected_overflow=0 exact_recv_per_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=81920 expected_recv=130572 expected_overflow=0 exact_recv_per_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130866 expected_overflow=0 exact_recv_per_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131135 expected_overflow=0 exact_recv_per_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130953 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131168 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=81920 expected_recv=130572 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130866 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=81920 expected_recv=131135 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=81920 expected_recv=131462 expected_overflow=0 exact_recv_per_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=81920 expected_recv=131123 expected_overflow=0 exact_recv_per_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130993 expected_overflow=0 exact_recv_per_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=81920 expected_recv=131462 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=81920 expected_recv=131123 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130993 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=81920 expected_recv=130957 expected_overflow=0 exact_recv_per_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130704 expected_overflow=0 exact_recv_per_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=81920 expected_recv=130832 expected_overflow=0 exact_recv_per_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=81920 expected_recv=131664 expected_overflow=0 exact_recv_per_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130811 expected_overflow=0 exact_recv_per_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=81920 expected_recv=130957 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130704 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=81920 expected_recv=130832 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=81920 expected_recv=131664 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=81920 expected_recv=130811 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=81920 expected_recv=131734 expected_overflow=0 exact_recv_per_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=81920 expected_recv=131170 expected_overflow=0 exact_recv_per_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=81920 expected_recv=130726 expected_overflow=0 exact_recv_per_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=81920 expected_recv=131734 actual_valid_recv=132992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=81920 expected_recv=131170 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=81920 expected_recv=130726 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132992 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131030 expected_overflow=0 exact_recv_per_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131073 expected_overflow=0 exact_recv_per_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=81920 expected_recv=131166 expected_overflow=0 exact_recv_per_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=81920 expected_recv=131031 expected_overflow=0 exact_recv_per_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131030 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=81920 expected_recv=130960 expected_overflow=0 exact_recv_per_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131073 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=81920 expected_recv=131166 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=81920 expected_recv=131031 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=81920 expected_recv=130960 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131237 expected_overflow=0 exact_recv_per_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=81920 expected_recv=130987 expected_overflow=0 exact_recv_per_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=81920 expected_recv=130712 expected_overflow=0 exact_recv_per_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131237 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=81920 expected_recv=130987 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=81920 expected_recv=130712 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=81920 expected_recv=131257 expected_overflow=0 exact_recv_per_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=81920 expected_recv=131385 expected_overflow=0 exact_recv_per_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131315 expected_overflow=0 exact_recv_per_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=81920 expected_recv=130808 expected_overflow=0 exact_recv_per_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=81920 expected_recv=131257 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131414 expected_overflow=0 exact_recv_per_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=81920 expected_recv=131385 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131315 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=81920 expected_recv=130808 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=81920 expected_recv=131414 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=81920 expected_recv=130967 expected_overflow=0 exact_recv_per_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=81920 expected_recv=130886 expected_overflow=0 exact_recv_per_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=81920 expected_recv=130731 expected_overflow=0 exact_recv_per_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=81920 expected_recv=130967 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=81920 expected_recv=130886 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=81920 expected_recv=130731 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=81920 expected_recv=131370 expected_overflow=0 exact_recv_per_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=81920 expected_recv=130944 expected_overflow=0 exact_recv_per_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=81920 expected_recv=131036 expected_overflow=0 exact_recv_per_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=81920 expected_recv=131370 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=81920 expected_recv=130944 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=81920 expected_recv=131036 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[65413, 65656, 65301, 65902, 65695, 65351, 65262, 65708] avg=65536.0000 max=65902 max_rank=3 min=65262 min_rank=6 imbalance_ratio=1.005585 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[65502, 65338, 65412, 65425, 65629, 65599, 65790, 65593] avg=65536.0000 max=65790 max_rank=6 min=65338 min_rank=1 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[65666, 65513, 65187, 65673, 65461, 65745, 65398, 65645] avg=65536.0000 max=65745 max_rank=5 min=65187 min_rank=2 imbalance_ratio=1.003189 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[65242, 65892, 65844, 65184, 65481, 65477, 65935, 65233] avg=65536.0000 max=65935 max_rank=6 min=65184 min_rank=3 imbalance_ratio=1.006088 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[65566, 65233, 65449, 65689, 65796, 65272, 65803, 65480] avg=65536.0000 max=65803 max_rank=6 min=65233 min_rank=1 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[65442, 65762, 66016, 65383, 65276, 65611, 65378, 65420] avg=65536.0000 max=66016 max_rank=2 min=65276 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[65512, 65364, 65288, 65951, 65606, 65814, 65370, 65383] avg=65536.0000 max=65951 max_rank=3 min=65288 min_rank=2 imbalance_ratio=1.006332 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[65686, 65717, 65327, 65237, 65597, 65255, 65987, 65482] avg=65536.0000 max=65987 max_rank=6 min=65237 min_rank=3 imbalance_ratio=1.006882 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[65536, 65181, 65914, 65774, 65229, 65456, 65594, 65604] avg=65536.0000 max=65914 max_rank=2 min=65181 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[65330, 65619, 65286, 65543, 65442, 65613, 65529, 65926] avg=65536.0000 max=65926 max_rank=7 min=65286 min_rank=2 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[65936, 65550, 65978, 65211, 65562, 65191, 65505, 65355] avg=65536.0000 max=65978 max_rank=2 min=65191 min_rank=5 imbalance_ratio=1.006744 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[65584, 65671, 65280, 65513, 65448, 65838, 65411, 65543] avg=65536.0000 max=65838 max_rank=5 min=65280 min_rank=2 imbalance_ratio=1.004608 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[65493, 65629, 65659, 65449, 65732, 65564, 65256, 65506] avg=65536.0000 max=65732 max_rank=4 min=65256 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[65493, 65809, 65629, 65427, 65496, 65376, 65646, 65412] avg=65536.0000 max=65809 max_rank=1 min=65376 min_rank=5 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[65666, 65203, 65616, 65810, 65855, 65476, 65512, 65150] avg=65536.0000 max=65855 max_rank=4 min=65150 min_rank=7 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[65570, 65588, 65477, 65548, 65460, 65691, 65666, 65288] avg=65536.0000 max=65691 max_rank=5 min=65288 min_rank=7 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=2095509 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509824 peak_demand=65955 required_capacity=1.006393 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=2097248 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508160 peak_demand=65901 required_capacity=1.005569 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=2098089 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506624 peak_demand=66034 required_capacity=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=2096246 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508288 peak_demand=65926 required_capacity=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=2096734 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508416 peak_demand=66016 required_capacity=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=2098877 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506368 peak_demand=65940 required_capacity=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=2097334 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507776 peak_demand=65987 required_capacity=1.006882 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=2097179 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508416 peak_demand=65953 required_capacity=1.006363 + [2026-08-30 05:16:38] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 474.8 | throughput per GPU (TFLOP/s/GPU): 702.2 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.311088E+01 | loss scale: 1.0 | grad norm: 39.880 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66560 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131000 expected_overflow=0 exact_recv_per_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131271 expected_overflow=0 exact_recv_per_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=81920 expected_recv=130938 expected_overflow=0 exact_recv_per_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=81920 expected_recv=130196 expected_overflow=0 exact_recv_per_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131202 expected_overflow=0 exact_recv_per_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131350 expected_overflow=0 exact_recv_per_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=81920 expected_recv=130938 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131122 expected_overflow=0 exact_recv_per_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131000 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131271 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131497 expected_overflow=0 exact_recv_per_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=81920 expected_recv=130196 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131202 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131350 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131122 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=81920 expected_recv=131497 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131166 expected_overflow=0 exact_recv_per_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131430 expected_overflow=0 exact_recv_per_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=81920 expected_recv=130801 expected_overflow=0 exact_recv_per_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131166 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131053 expected_overflow=0 exact_recv_per_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131430 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131035 expected_overflow=0 exact_recv_per_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=81920 expected_recv=130801 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131251 expected_overflow=0 exact_recv_per_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131053 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131035 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=81920 expected_recv=130814 expected_overflow=0 exact_recv_per_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131026 expected_overflow=0 exact_recv_per_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131251 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=81920 expected_recv=130814 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=81920 expected_recv=131026 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131022 expected_overflow=0 exact_recv_per_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131407 expected_overflow=0 exact_recv_per_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131022 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131285 expected_overflow=0 exact_recv_per_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131407 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131225 expected_overflow=0 exact_recv_per_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131285 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=81920 expected_recv=130897 expected_overflow=0 exact_recv_per_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131105 expected_overflow=0 exact_recv_per_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131225 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=81920 expected_recv=130858 expected_overflow=0 exact_recv_per_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=81920 expected_recv=130777 expected_overflow=0 exact_recv_per_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=81920 expected_recv=130897 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=81920 expected_recv=131105 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=81920 expected_recv=130858 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=81920 expected_recv=130777 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=81920 expected_recv=130566 expected_overflow=0 exact_recv_per_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131349 expected_overflow=0 exact_recv_per_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=81920 expected_recv=130566 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131349 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=81920 expected_recv=130922 expected_overflow=0 exact_recv_per_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=81920 expected_recv=130917 expected_overflow=0 exact_recv_per_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=81920 expected_recv=130922 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=81920 expected_recv=130901 expected_overflow=0 exact_recv_per_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131114 expected_overflow=0 exact_recv_per_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131499 expected_overflow=0 exact_recv_per_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=81920 expected_recv=130917 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=81920 expected_recv=130901 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131308 expected_overflow=0 exact_recv_per_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131114 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131499 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=81920 expected_recv=131308 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130896 expected_overflow=0 exact_recv_per_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=81920 expected_recv=131080 expected_overflow=0 exact_recv_per_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130896 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=81920 expected_recv=131080 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=81920 expected_recv=131158 expected_overflow=0 exact_recv_per_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130776 expected_overflow=0 exact_recv_per_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=81920 expected_recv=131158 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=81920 expected_recv=131223 expected_overflow=0 exact_recv_per_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130933 expected_overflow=0 exact_recv_per_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130776 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=81920 expected_recv=131198 expected_overflow=0 exact_recv_per_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=81920 expected_recv=131312 expected_overflow=0 exact_recv_per_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=81920 expected_recv=131223 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=81920 expected_recv=130933 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=81920 expected_recv=131198 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=81920 expected_recv=131312 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=81920 expected_recv=130839 expected_overflow=0 exact_recv_per_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=81920 expected_recv=130838 expected_overflow=0 exact_recv_per_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=81920 expected_recv=130839 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=81920 expected_recv=130838 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131051 expected_overflow=0 exact_recv_per_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=81920 expected_recv=130681 expected_overflow=0 exact_recv_per_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131051 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=81920 expected_recv=130681 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=81920 expected_recv=130861 expected_overflow=0 exact_recv_per_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131232 expected_overflow=0 exact_recv_per_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131305 expected_overflow=0 exact_recv_per_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=81920 expected_recv=130861 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131769 expected_overflow=0 exact_recv_per_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131232 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131305 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=81920 expected_recv=130532 expected_overflow=0 exact_recv_per_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=81920 expected_recv=131769 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131567 expected_overflow=0 exact_recv_per_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=81920 expected_recv=130532 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131172 expected_overflow=0 exact_recv_per_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131567 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=81920 expected_recv=130974 expected_overflow=0 exact_recv_per_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131172 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=81920 expected_recv=130974 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131008 expected_overflow=0 exact_recv_per_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131289 expected_overflow=0 exact_recv_per_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131073 expected_overflow=0 exact_recv_per_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131008 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131289 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=81920 expected_recv=130961 expected_overflow=0 exact_recv_per_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=81920 expected_recv=131073 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=81920 expected_recv=131474 expected_overflow=0 exact_recv_per_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=81920 expected_recv=130961 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=81920 expected_recv=130978 expected_overflow=0 exact_recv_per_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=81920 expected_recv=131474 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=81920 expected_recv=130832 expected_overflow=0 exact_recv_per_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=81920 expected_recv=130978 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=81920 expected_recv=131306 expected_overflow=0 exact_recv_per_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=81920 expected_recv=130832 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=81920 expected_recv=131306 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=81920 expected_recv=131346 expected_overflow=0 exact_recv_per_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=81920 expected_recv=130665 expected_overflow=0 exact_recv_per_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=81920 expected_recv=130964 expected_overflow=0 exact_recv_per_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=81920 expected_recv=131346 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=81920 expected_recv=130665 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=81920 expected_recv=130964 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=81920 expected_recv=131011 expected_overflow=0 exact_recv_per_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=81920 expected_recv=131011 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=81920 expected_recv=130884 expected_overflow=0 exact_recv_per_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=81920 expected_recv=131422 expected_overflow=0 exact_recv_per_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=81920 expected_recv=130884 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=81920 expected_recv=131422 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=81920 expected_recv=131272 expected_overflow=0 exact_recv_per_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=81920 expected_recv=131020 expected_overflow=0 exact_recv_per_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=81920 expected_recv=131272 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=81920 expected_recv=131020 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=81920 expected_recv=131376 expected_overflow=0 exact_recv_per_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=81920 expected_recv=130925 expected_overflow=0 exact_recv_per_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=81920 expected_recv=130914 expected_overflow=0 exact_recv_per_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=81920 expected_recv=131376 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=81920 expected_recv=130925 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=81920 expected_recv=130763 expected_overflow=0 exact_recv_per_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=81920 expected_recv=130914 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=81920 expected_recv=130763 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=81920 expected_recv=130705 expected_overflow=0 exact_recv_per_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131408 expected_overflow=0 exact_recv_per_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=81920 expected_recv=130705 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131408 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131415 expected_overflow=0 exact_recv_per_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131199 expected_overflow=0 exact_recv_per_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131055 expected_overflow=0 exact_recv_per_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131415 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131199 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131055 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=81920 expected_recv=130963 expected_overflow=0 exact_recv_per_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131145 expected_overflow=0 exact_recv_per_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=81920 expected_recv=130686 expected_overflow=0 exact_recv_per_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=81920 expected_recv=130963 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=81920 expected_recv=131145 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=81920 expected_recv=130686 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=81920 expected_recv=130775 expected_overflow=0 exact_recv_per_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=81920 expected_recv=130396 expected_overflow=0 exact_recv_per_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=81920 expected_recv=130775 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=81920 expected_recv=130396 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131127 expected_overflow=0 exact_recv_per_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131301 expected_overflow=0 exact_recv_per_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131751 expected_overflow=0 exact_recv_per_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131127 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131301 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131751 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131160 expected_overflow=0 exact_recv_per_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131090 expected_overflow=0 exact_recv_per_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=81920 expected_recv=130976 expected_overflow=0 exact_recv_per_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131160 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=81920 expected_recv=131090 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=81920 expected_recv=130976 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=81920 expected_recv=130992 expected_overflow=0 exact_recv_per_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=81920 expected_recv=130883 expected_overflow=0 exact_recv_per_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=81920 expected_recv=130992 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=81920 expected_recv=130883 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=81920 expected_recv=131417 expected_overflow=0 exact_recv_per_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=81920 expected_recv=131172 expected_overflow=0 exact_recv_per_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=81920 expected_recv=131425 expected_overflow=0 exact_recv_per_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=81920 expected_recv=131417 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=81920 expected_recv=131172 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=81920 expected_recv=131425 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=81920 expected_recv=130619 expected_overflow=0 exact_recv_per_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=81920 expected_recv=130854 expected_overflow=0 exact_recv_per_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=81920 expected_recv=131214 expected_overflow=0 exact_recv_per_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=81920 expected_recv=130619 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=81920 expected_recv=130854 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=81920 expected_recv=131214 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131045 expected_overflow=0 exact_recv_per_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131160 expected_overflow=0 exact_recv_per_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131045 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131160 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131261 expected_overflow=0 exact_recv_per_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131261 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=81920 expected_recv=130728 expected_overflow=0 exact_recv_per_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131084 expected_overflow=0 exact_recv_per_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=81920 expected_recv=130728 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131084 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=81920 expected_recv=130634 expected_overflow=0 exact_recv_per_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=81920 expected_recv=130976 expected_overflow=0 exact_recv_per_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131688 expected_overflow=0 exact_recv_per_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=81920 expected_recv=130634 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=81920 expected_recv=130976 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=81920 expected_recv=131688 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=81920 expected_recv=131432 expected_overflow=0 exact_recv_per_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=81920 expected_recv=131035 expected_overflow=0 exact_recv_per_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=81920 expected_recv=131102 expected_overflow=0 exact_recv_per_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=81920 expected_recv=131432 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=81920 expected_recv=131035 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=81920 expected_recv=131102 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=81920 expected_recv=131420 expected_overflow=0 exact_recv_per_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=81920 expected_recv=131468 expected_overflow=0 exact_recv_per_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=81920 expected_recv=131420 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=81920 expected_recv=130827 expected_overflow=0 exact_recv_per_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=81920 expected_recv=130362 expected_overflow=0 exact_recv_per_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=81920 expected_recv=131468 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=81920 expected_recv=130827 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=81920 expected_recv=130930 expected_overflow=0 exact_recv_per_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=81920 expected_recv=130362 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131238 expected_overflow=0 exact_recv_per_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131544 expected_overflow=0 exact_recv_per_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=81920 expected_recv=130930 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131238 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131544 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131207 expected_overflow=0 exact_recv_per_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131207 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=81920 expected_recv=130863 expected_overflow=0 exact_recv_per_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=81920 expected_recv=130305 expected_overflow=0 exact_recv_per_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131080 expected_overflow=0 exact_recv_per_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=81920 expected_recv=130863 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=81920 expected_recv=130849 expected_overflow=0 exact_recv_per_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=81920 expected_recv=130305 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131080 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131490 expected_overflow=0 exact_recv_per_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=81920 expected_recv=130849 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=81920 expected_recv=130675 expected_overflow=0 exact_recv_per_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=81920 expected_recv=131490 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131083 expected_overflow=0 exact_recv_per_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=81920 expected_recv=130675 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131083 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=81920 expected_recv=130890 expected_overflow=0 exact_recv_per_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=81920 expected_recv=130890 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131101 expected_overflow=0 exact_recv_per_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131038 expected_overflow=0 exact_recv_per_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131247 expected_overflow=0 exact_recv_per_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131101 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=81920 expected_recv=130896 expected_overflow=0 exact_recv_per_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131038 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131646 expected_overflow=0 exact_recv_per_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131247 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=81920 expected_recv=130896 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=81920 expected_recv=131646 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[65855, 65508, 65394, 65171, 65745, 65611, 65671, 65333] avg=65536.0000 max=65855 max_rank=0 min=65171 min_rank=3 imbalance_ratio=1.004868 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[65849, 65420, 65639, 65652, 65297, 65494, 65572, 65365] avg=65536.0000 max=65849 max_rank=0 min=65297 min_rank=4 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[65735, 65757, 65385, 65666, 65256, 65507, 65608, 65374] avg=65536.0000 max=65757 max_rank=1 min=65256 min_rank=4 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[65748, 65294, 65522, 65463, 65559, 65361, 65689, 65652] avg=65536.0000 max=65748 max_rank=0 min=65294 min_rank=1 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65266, 66277] avg=65536.0000 max=66277 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.011307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65266, 66277] avg=65536.0000 max=66277 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.011307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65266, 66277] avg=65536.0000 max=66277 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.011307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65266, 66277] avg=65536.0000 max=66277 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.011307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] avg=65536.0000 max=66192 max_rank=7 min=65300 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.001282 moved_to_replicas=85 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[65404, 65436, 65725, 65558, 65783, 65281, 65383, 65718] avg=65536.0000 max=65783 max_rank=4 min=65281 min_rank=5 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] avg=65536.0000 max=66192 max_rank=7 min=65300 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.001282 moved_to_replicas=85 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65266, 66277] avg=65536.0000 max=66277 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.011307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] avg=65536.0000 max=66192 max_rank=7 min=65300 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.001282 moved_to_replicas=85 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] avg=65536.0000 max=66192 max_rank=7 min=65300 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.001282 moved_to_replicas=85 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65266, 66277] avg=65536.0000 max=66277 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.011307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] avg=65536.0000 max=66192 max_rank=7 min=65300 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.001282 moved_to_replicas=85 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65266, 66277] avg=65536.0000 max=66277 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.011307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65266, 66277] avg=65536.0000 max=66277 max_rank=7 min=65266 min_rank=6 imbalance_ratio=1.011307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] avg=65536.0000 max=66192 max_rank=7 min=65300 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.001282 moved_to_replicas=85 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] avg=65536.0000 max=66192 max_rank=7 min=65300 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.001282 moved_to_replicas=85 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[65361, 65300, 65379, 65409, 65891, 65405, 65351, 66192] avg=65536.0000 max=66192 max_rank=7 min=65300 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=85 max_load_reduction_pct=0.001282 moved_to_replicas=85 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65486, 65335, 65614, 65461, 65809, 65737, 65318, 65528] avg=65536.0000 max=65809 max_rank=4 min=65318 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[65703, 65316, 65521, 65731, 65232, 65931, 65215, 65639] avg=65536.0000 max=65931 max_rank=5 min=65215 min_rank=6 imbalance_ratio=1.006027 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[65656, 65553, 65530, 65322, 65642, 65681, 65501, 65403] avg=65536.0000 max=65681 max_rank=5 min=65322 min_rank=3 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=91 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=91 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65647, 65608, 65174, 65840, 65574, 65512, 65287, 65646] avg=65536.0000 max=65840 max_rank=3 min=65174 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=91 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=91 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=91 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=91 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=91 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[65509, 65493, 65073, 65683, 65559, 65864, 65725, 65382] avg=65536.0000 max=65864 max_rank=5 min=65073 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=91 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[65419, 65496, 65232, 65459, 65843, 65733, 65237, 65869] avg=65536.0000 max=65869 max_rank=7 min=65232 min_rank=2 imbalance_ratio=1.005081 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[65646, 65543, 65296, 65735, 65473, 65592, 65439, 65564] avg=65536.0000 max=65735 max_rank=3 min=65296 min_rank=2 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[65485, 65648, 65710, 65364, 65882, 65483, 65415, 65301] avg=65536.0000 max=65882 max_rank=4 min=65301 min_rank=7 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[65534, 65257, 65766, 65949, 65097, 65658, 65721, 65306] avg=65536.0000 max=65949 max_rank=3 min=65097 min_rank=4 imbalance_ratio=1.006302 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[65350, 65575, 65560, 65782, 65527, 65562, 65503, 65429] avg=65536.0000 max=65782 max_rank=3 min=65350 min_rank=0 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=2097707 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508288 peak_demand=65855 required_capacity=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=2096008 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=510720 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=2097548 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508416 peak_demand=65837 required_capacity=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=2097573 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508544 peak_demand=65931 required_capacity=1.006027 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=2096052 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509824 peak_demand=65953 required_capacity=1.006363 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=2098554 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=505856 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=2096539 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508160 peak_demand=65954 required_capacity=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=2097235 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507264 peak_demand=65953 required_capacity=1.006363 + [2026-08-30 05:16:38] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 471.7 | throughput per GPU (TFLOP/s/GPU): 706.8 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.035518E+01 | loss scale: 1.0 | grad norm: 15.227 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66560 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131283 expected_overflow=0 exact_recv_per_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131069 expected_overflow=0 exact_recv_per_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131113 expected_overflow=0 exact_recv_per_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131034 expected_overflow=0 exact_recv_per_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131094 expected_overflow=0 exact_recv_per_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131320 expected_overflow=0 exact_recv_per_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131283 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131069 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131113 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=81920 expected_recv=130658 expected_overflow=0 exact_recv_per_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131034 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131094 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131005 expected_overflow=0 exact_recv_per_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131320 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=81920 expected_recv=130658 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=81920 expected_recv=131005 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131433 expected_overflow=0 exact_recv_per_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131378 expected_overflow=0 exact_recv_per_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=81920 expected_recv=130692 expected_overflow=0 exact_recv_per_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131355 expected_overflow=0 exact_recv_per_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131433 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131378 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=81920 expected_recv=130816 expected_overflow=0 exact_recv_per_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=81920 expected_recv=130692 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131355 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=81920 expected_recv=130954 expected_overflow=0 exact_recv_per_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131117 expected_overflow=0 exact_recv_per_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=81920 expected_recv=130816 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=81920 expected_recv=130954 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=81920 expected_recv=131117 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=81920 expected_recv=130831 expected_overflow=0 exact_recv_per_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=81920 expected_recv=130831 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130874 expected_overflow=0 exact_recv_per_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130933 expected_overflow=0 exact_recv_per_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131212 expected_overflow=0 exact_recv_per_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130874 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131235 expected_overflow=0 exact_recv_per_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130611 expected_overflow=0 exact_recv_per_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130933 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131212 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130724 expected_overflow=0 exact_recv_per_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131235 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131661 expected_overflow=0 exact_recv_per_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130611 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=81920 expected_recv=130724 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131661 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131326 expected_overflow=0 exact_recv_per_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=81920 expected_recv=131700 expected_overflow=0 exact_recv_per_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=81920 expected_recv=131326 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=81920 expected_recv=131356 expected_overflow=0 exact_recv_per_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130727 expected_overflow=0 exact_recv_per_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=81920 expected_recv=131697 expected_overflow=0 exact_recv_per_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=81920 expected_recv=131700 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130493 expected_overflow=0 exact_recv_per_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=81920 expected_recv=131356 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130727 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130591 expected_overflow=0 exact_recv_per_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=81920 expected_recv=131697 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130493 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=81920 expected_recv=131233 expected_overflow=0 exact_recv_per_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130591 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=81920 expected_recv=131233 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130779 expected_overflow=0 exact_recv_per_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130999 expected_overflow=0 exact_recv_per_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130744 expected_overflow=0 exact_recv_per_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=81920 expected_recv=130779 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131474 expected_overflow=0 exact_recv_per_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130999 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130744 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131474 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130486 expected_overflow=0 exact_recv_per_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131106 expected_overflow=0 exact_recv_per_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130486 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130967 expected_overflow=0 exact_recv_per_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131075 expected_overflow=0 exact_recv_per_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131106 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=81920 expected_recv=130967 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131075 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131725 expected_overflow=0 exact_recv_per_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=81920 expected_recv=130994 expected_overflow=0 exact_recv_per_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=81920 expected_recv=130967 expected_overflow=0 exact_recv_per_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=81920 expected_recv=131725 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=81920 expected_recv=130994 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=81920 expected_recv=130929 expected_overflow=0 exact_recv_per_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=81920 expected_recv=130967 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=81920 expected_recv=131231 expected_overflow=0 exact_recv_per_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=81920 expected_recv=130929 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=81920 expected_recv=131268 expected_overflow=0 exact_recv_per_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=81920 expected_recv=130683 expected_overflow=0 exact_recv_per_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=81920 expected_recv=131432 expected_overflow=0 exact_recv_per_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=81920 expected_recv=131231 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=81920 expected_recv=131268 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=81920 expected_recv=131432 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=81920 expected_recv=130683 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=81920 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=81920 expected_recv=130888 expected_overflow=0 exact_recv_per_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=81920 expected_recv=131072 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=81920 expected_recv=130667 expected_overflow=0 exact_recv_per_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=81920 expected_recv=130888 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131068 expected_overflow=0 exact_recv_per_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=81920 expected_recv=130667 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131373 expected_overflow=0 exact_recv_per_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131068 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=81920 expected_recv=130500 expected_overflow=0 exact_recv_per_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131628 expected_overflow=0 exact_recv_per_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131170 expected_overflow=0 exact_recv_per_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131373 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=81920 expected_recv=130500 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131628 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131170 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131282 expected_overflow=0 exact_recv_per_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131440 expected_overflow=0 exact_recv_per_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=81920 expected_recv=130990 expected_overflow=0 exact_recv_per_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=81920 expected_recv=131282 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131440 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=81920 expected_recv=130990 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131199 expected_overflow=0 exact_recv_per_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131082 expected_overflow=0 exact_recv_per_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131199 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131189 expected_overflow=0 exact_recv_per_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131056 expected_overflow=0 exact_recv_per_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131082 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=81920 expected_recv=130857 expected_overflow=0 exact_recv_per_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131189 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=81920 expected_recv=131056 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=81920 expected_recv=130857 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=81920 expected_recv=130763 expected_overflow=0 exact_recv_per_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=81920 expected_recv=130900 expected_overflow=0 exact_recv_per_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131026 expected_overflow=0 exact_recv_per_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=81920 expected_recv=130763 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=81920 expected_recv=130900 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131026 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131497 expected_overflow=0 exact_recv_per_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=81920 expected_recv=130617 expected_overflow=0 exact_recv_per_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131497 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=81920 expected_recv=130851 expected_overflow=0 exact_recv_per_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131156 expected_overflow=0 exact_recv_per_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=81920 expected_recv=130617 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131121 expected_overflow=0 exact_recv_per_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=81920 expected_recv=130851 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131156 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131121 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131028 expected_overflow=0 exact_recv_per_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131408 expected_overflow=0 exact_recv_per_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131120 expected_overflow=0 exact_recv_per_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131028 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=81920 expected_recv=131408 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131120 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131158 expected_overflow=0 exact_recv_per_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=81920 expected_recv=130862 expected_overflow=0 exact_recv_per_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=81920 expected_recv=130869 expected_overflow=0 exact_recv_per_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131158 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131206 expected_overflow=0 exact_recv_per_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131286 expected_overflow=0 exact_recv_per_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=81920 expected_recv=130862 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=81920 expected_recv=130869 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131206 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131286 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=81920 expected_recv=130799 expected_overflow=0 exact_recv_per_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131047 expected_overflow=0 exact_recv_per_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=81920 expected_recv=130799 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=81920 expected_recv=130344 expected_overflow=0 exact_recv_per_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=81920 expected_recv=131047 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=81920 expected_recv=131590 expected_overflow=0 exact_recv_per_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=81920 expected_recv=130344 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=81920 expected_recv=131314 expected_overflow=0 exact_recv_per_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=81920 expected_recv=131586 expected_overflow=0 exact_recv_per_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=81920 expected_recv=131766 expected_overflow=0 exact_recv_per_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=81920 expected_recv=131590 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=81920 expected_recv=130615 expected_overflow=0 exact_recv_per_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=81920 expected_recv=131314 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=81920 expected_recv=131586 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=81920 expected_recv=131766 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=81920 expected_recv=130615 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=81920 expected_recv=130998 expected_overflow=0 exact_recv_per_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=81920 expected_recv=130562 expected_overflow=0 exact_recv_per_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=81920 expected_recv=130998 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131512 expected_overflow=0 exact_recv_per_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=81920 expected_recv=130562 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131512 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=81920 expected_recv=130899 expected_overflow=0 exact_recv_per_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131167 expected_overflow=0 exact_recv_per_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131408 expected_overflow=0 exact_recv_per_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=81920 expected_recv=130899 expected_overflow=0 exact_recv_per_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=81920 expected_recv=130899 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131167 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131408 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131116 expected_overflow=0 exact_recv_per_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=81920 expected_recv=130899 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=81920 expected_recv=131116 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131061 expected_overflow=0 exact_recv_per_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131061 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=81920 expected_recv=130890 expected_overflow=0 exact_recv_per_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=81920 expected_recv=130577 expected_overflow=0 exact_recv_per_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=81920 expected_recv=130890 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=81920 expected_recv=130577 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131245 expected_overflow=0 exact_recv_per_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=81920 expected_recv=130797 expected_overflow=0 exact_recv_per_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131040 expected_overflow=0 exact_recv_per_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131245 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=81920 expected_recv=130797 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131139 expected_overflow=0 exact_recv_per_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131040 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131257 expected_overflow=0 exact_recv_per_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131139 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131257 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=81920 expected_recv=130727 expected_overflow=0 exact_recv_per_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=81920 expected_recv=130727 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=81920 expected_recv=130965 expected_overflow=0 exact_recv_per_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=81920 expected_recv=130965 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131147 expected_overflow=0 exact_recv_per_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131486 expected_overflow=0 exact_recv_per_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131001 expected_overflow=0 exact_recv_per_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=81920 expected_recv=131147 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131087 expected_overflow=0 exact_recv_per_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131486 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131106 expected_overflow=0 exact_recv_per_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131344 expected_overflow=0 exact_recv_per_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131001 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131087 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131106 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130893 expected_overflow=0 exact_recv_per_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=81920 expected_recv=131344 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130893 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130961 expected_overflow=0 exact_recv_per_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130961 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=81920 expected_recv=130860 expected_overflow=0 exact_recv_per_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=81920 expected_recv=131054 expected_overflow=0 exact_recv_per_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130908 expected_overflow=0 exact_recv_per_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130992 expected_overflow=0 exact_recv_per_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=81920 expected_recv=130860 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=81920 expected_recv=131054 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=81920 expected_recv=131653 expected_overflow=0 exact_recv_per_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130908 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=81920 expected_recv=131164 expected_overflow=0 exact_recv_per_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130992 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=81920 expected_recv=131653 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130776 expected_overflow=0 exact_recv_per_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=81920 expected_recv=131164 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130776 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130995 expected_overflow=0 exact_recv_per_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130995 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130951 expected_overflow=0 exact_recv_per_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130684 expected_overflow=0 exact_recv_per_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=81920 expected_recv=130951 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130888 expected_overflow=0 exact_recv_per_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131514 expected_overflow=0 exact_recv_per_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130684 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131088 expected_overflow=0 exact_recv_per_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131291 expected_overflow=0 exact_recv_per_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=81920 expected_recv=130888 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131514 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131088 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131291 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131340 expected_overflow=0 exact_recv_per_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=81920 expected_recv=131340 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[65431, 65579, 65692, 65511, 65455, 65280, 65532, 65808] avg=65536.0000 max=65808 max_rank=7 min=65280 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[65723, 65425, 65704, 65236, 65664, 65644, 65401, 65491] avg=65536.0000 max=65723 max_rank=0 min=65236 min_rank=3 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[65323, 65713, 65158, 65109, 65438, 65891, 65864, 65792] avg=65536.0000 max=65891 max_rank=5 min=65109 min_rank=3 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[65855, 65616, 65414, 65023, 65282, 65889, 65451, 65758] avg=65536.0000 max=65889 max_rank=5 min=65023 min_rank=3 imbalance_ratio=1.005386 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65471, 65245, 65827, 65745, 65374, 65362, 65689, 65575] avg=65536.0000 max=65827 max_rank=2 min=65245 min_rank=1 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[65732, 65604, 65205, 65528, 65593, 65513, 65625, 65488] avg=65536.0000 max=65732 max_rank=0 min=65205 min_rank=2 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[65331, 65470, 65465, 65071, 65674, 65780, 65734, 65763] avg=65536.0000 max=65780 max_rank=5 min=65071 min_rank=3 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[65647, 65422, 65834, 65473, 65420, 65615, 65384, 65493] avg=65536.0000 max=65834 max_rank=2 min=65384 min_rank=6 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[65448, 65476, 65533, 65790, 65523, 65324, 65645, 65549] avg=65536.0000 max=65790 max_rank=3 min=65324 min_rank=5 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65430, 65673, 65582, 65313, 65514, 65570, 65532, 65674] avg=65536.0000 max=65674 max_rank=7 min=65313 min_rank=3 imbalance_ratio=1.002106 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[65282, 65460, 65722, 65763, 65730, 65683, 65337, 65311] avg=65536.0000 max=65763 max_rank=3 min=65282 min_rank=0 imbalance_ratio=1.003464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65475, 65614, 65827, 65191, 65716, 65418, 65419, 65628] avg=65536.0000 max=65827 max_rank=2 min=65191 min_rank=3 imbalance_ratio=1.004440 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[65558, 65488, 65391, 65499, 65206, 65859, 65442, 65845] avg=65536.0000 max=65859 max_rank=5 min=65206 min_rank=4 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[65640, 65387, 65325, 65486, 65552, 65857, 65196, 65845] avg=65536.0000 max=65857 max_rank=5 min=65196 min_rank=6 imbalance_ratio=1.004898 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[65425, 65658, 65518, 65587, 65362, 65243, 65578, 65917] avg=65536.0000 max=65917 max_rank=7 min=65243 min_rank=5 imbalance_ratio=1.005814 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[65278, 65605, 65376, 65660, 65583, 65572, 65581, 65633] avg=65536.0000 max=65660 max_rank=3 min=65278 min_rank=0 imbalance_ratio=1.001892 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=2096579 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508416 peak_demand=65855 required_capacity=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=2095554 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509824 peak_demand=65898 required_capacity=1.005524 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=2098201 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507008 peak_demand=65903 required_capacity=1.005600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=2098202 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506752 peak_demand=65917 required_capacity=1.005814 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=2096481 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508032 peak_demand=65954 required_capacity=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=2098567 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506624 peak_demand=65964 required_capacity=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=2096957 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507648 peak_demand=66066 required_capacity=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=2096675 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508672 peak_demand=66036 required_capacity=1.007629 + [2026-08-30 05:16:39] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 474.7 | throughput per GPU (TFLOP/s/GPU): 702.3 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.030773E+01 | loss scale: 1.0 | grad norm: 29.271 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66560 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=81920 expected_recv=131047 expected_overflow=0 exact_recv_per_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=81920 expected_recv=131117 expected_overflow=0 exact_recv_per_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=81920 expected_recv=131821 expected_overflow=0 exact_recv_per_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=81920 expected_recv=131277 expected_overflow=0 exact_recv_per_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=81920 expected_recv=131047 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=81920 expected_recv=130611 expected_overflow=0 exact_recv_per_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=81920 expected_recv=130487 expected_overflow=0 exact_recv_per_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=81920 expected_recv=131578 expected_overflow=0 exact_recv_per_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=81920 expected_recv=131117 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=81920 expected_recv=131821 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=81920 expected_recv=131277 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=81920 expected_recv=130638 expected_overflow=0 exact_recv_per_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=81920 expected_recv=130611 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=81920 expected_recv=130487 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=81920 expected_recv=131578 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=81920 expected_recv=130638 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131218 expected_overflow=0 exact_recv_per_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131504 expected_overflow=0 exact_recv_per_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131334 expected_overflow=0 exact_recv_per_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131218 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=81920 expected_recv=130510 expected_overflow=0 exact_recv_per_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131504 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131334 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=81920 expected_recv=130873 expected_overflow=0 exact_recv_per_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=81920 expected_recv=130510 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131053 expected_overflow=0 exact_recv_per_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=81920 expected_recv=130873 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131340 expected_overflow=0 exact_recv_per_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131053 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=81920 expected_recv=130744 expected_overflow=0 exact_recv_per_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131182 expected_overflow=0 exact_recv_per_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=81920 expected_recv=131340 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=81920 expected_recv=130744 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131419 expected_overflow=0 exact_recv_per_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131291 expected_overflow=0 exact_recv_per_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131182 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131120 expected_overflow=0 exact_recv_per_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131419 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131291 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131120 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=81920 expected_recv=130641 expected_overflow=0 exact_recv_per_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=81920 expected_recv=130494 expected_overflow=0 exact_recv_per_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=81920 expected_recv=130641 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131225 expected_overflow=0 exact_recv_per_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=81920 expected_recv=130494 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131204 expected_overflow=0 exact_recv_per_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=81920 expected_recv=131155 expected_overflow=0 exact_recv_per_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131225 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=81920 expected_recv=131223 expected_overflow=0 exact_recv_per_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=81920 expected_recv=131204 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=81920 expected_recv=131155 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130983 expected_overflow=0 exact_recv_per_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=81920 expected_recv=131223 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130993 expected_overflow=0 exact_recv_per_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130983 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=81920 expected_recv=131048 expected_overflow=0 exact_recv_per_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130993 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130926 expected_overflow=0 exact_recv_per_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=81920 expected_recv=131048 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130864 expected_overflow=0 exact_recv_per_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130926 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=81920 expected_recv=131384 expected_overflow=0 exact_recv_per_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=81920 expected_recv=130864 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=81920 expected_recv=130924 expected_overflow=0 exact_recv_per_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131582 expected_overflow=0 exact_recv_per_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=81920 expected_recv=131384 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=81920 expected_recv=130924 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131582 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131025 expected_overflow=0 exact_recv_per_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131073 expected_overflow=0 exact_recv_per_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131025 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131073 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=81920 expected_recv=130976 expected_overflow=0 exact_recv_per_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=81920 expected_recv=130976 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=81920 expected_recv=130753 expected_overflow=0 exact_recv_per_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131211 expected_overflow=0 exact_recv_per_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=81920 expected_recv=130753 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131211 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131496 expected_overflow=0 exact_recv_per_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131003 expected_overflow=0 exact_recv_per_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131032 expected_overflow=0 exact_recv_per_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131084 expected_overflow=0 exact_recv_per_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=81920 expected_recv=130709 expected_overflow=0 exact_recv_per_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131496 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131003 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=81920 expected_recv=131032 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=81920 expected_recv=130994 expected_overflow=0 exact_recv_per_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131084 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=81920 expected_recv=130709 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=81920 expected_recv=130994 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131376 expected_overflow=0 exact_recv_per_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=81920 expected_recv=130806 expected_overflow=0 exact_recv_per_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131376 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=81920 expected_recv=130806 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131402 expected_overflow=0 exact_recv_per_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131337 expected_overflow=0 exact_recv_per_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=81920 expected_recv=130662 expected_overflow=0 exact_recv_per_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131108 expected_overflow=0 exact_recv_per_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131206 expected_overflow=0 exact_recv_per_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131337 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131402 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=81920 expected_recv=130662 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=81920 expected_recv=130919 expected_overflow=0 exact_recv_per_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=81920 expected_recv=131108 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131206 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=81920 expected_recv=130919 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131012 expected_overflow=0 exact_recv_per_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131185 expected_overflow=0 exact_recv_per_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131012 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=81920 expected_recv=131185 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131129 expected_overflow=0 exact_recv_per_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131059 expected_overflow=0 exact_recv_per_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131129 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131040 expected_overflow=0 exact_recv_per_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131236 expected_overflow=0 exact_recv_per_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=81920 expected_recv=130853 expected_overflow=0 exact_recv_per_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131059 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131220 expected_overflow=0 exact_recv_per_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131040 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131236 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=81920 expected_recv=130853 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131220 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131371 expected_overflow=0 exact_recv_per_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=81920 expected_recv=130615 expected_overflow=0 exact_recv_per_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131131 expected_overflow=0 exact_recv_per_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=81920 expected_recv=131371 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=81920 expected_recv=130615 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131131 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131054 expected_overflow=0 exact_recv_per_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=81920 expected_recv=130537 expected_overflow=0 exact_recv_per_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131161 expected_overflow=0 exact_recv_per_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131427 expected_overflow=0 exact_recv_per_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=81920 expected_recv=130906 expected_overflow=0 exact_recv_per_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131054 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=81920 expected_recv=130537 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131161 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131427 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=81920 expected_recv=130906 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=81920 expected_recv=130926 expected_overflow=0 exact_recv_per_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=81920 expected_recv=130985 expected_overflow=0 exact_recv_per_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=81920 expected_recv=130926 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130959 expected_overflow=0 exact_recv_per_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=81920 expected_recv=130985 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130959 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131454 expected_overflow=0 exact_recv_per_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131063 expected_overflow=0 exact_recv_per_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130965 expected_overflow=0 exact_recv_per_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130862 expected_overflow=0 exact_recv_per_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131454 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131063 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131355 expected_overflow=0 exact_recv_per_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130965 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130862 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=81920 expected_recv=131355 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130733 expected_overflow=0 exact_recv_per_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131488 expected_overflow=0 exact_recv_per_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=81920 expected_recv=130807 expected_overflow=0 exact_recv_per_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=81920 expected_recv=130733 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131488 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=81920 expected_recv=130807 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131376 expected_overflow=0 exact_recv_per_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131288 expected_overflow=0 exact_recv_per_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131323 expected_overflow=0 exact_recv_per_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131376 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=81920 expected_recv=130956 expected_overflow=0 exact_recv_per_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131288 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131323 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131052 expected_overflow=0 exact_recv_per_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=81920 expected_recv=130956 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=81920 expected_recv=130590 expected_overflow=0 exact_recv_per_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131631 expected_overflow=0 exact_recv_per_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131452 expected_overflow=0 exact_recv_per_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=81920 expected_recv=131052 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=81920 expected_recv=130590 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=81920 expected_recv=131631 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131452 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=81920 expected_recv=130898 expected_overflow=0 exact_recv_per_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131536 expected_overflow=0 exact_recv_per_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=81920 expected_recv=130898 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131184 expected_overflow=0 exact_recv_per_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131536 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=81920 expected_recv=130919 expected_overflow=0 exact_recv_per_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131184 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131036 expected_overflow=0 exact_recv_per_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=81920 expected_recv=130919 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=81920 expected_recv=130605 expected_overflow=0 exact_recv_per_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=81920 expected_recv=130632 expected_overflow=0 exact_recv_per_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=81920 expected_recv=130820 expected_overflow=0 exact_recv_per_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=81920 expected_recv=131036 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=81920 expected_recv=130605 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=81920 expected_recv=130632 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=81920 expected_recv=130820 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=81920 expected_recv=130747 expected_overflow=0 exact_recv_per_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131066 expected_overflow=0 exact_recv_per_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=81920 expected_recv=130793 expected_overflow=0 exact_recv_per_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=81920 expected_recv=130747 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131066 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=81920 expected_recv=130793 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131621 expected_overflow=0 exact_recv_per_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131314 expected_overflow=0 exact_recv_per_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131621 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=81920 expected_recv=130919 expected_overflow=0 exact_recv_per_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=81920 expected_recv=130860 expected_overflow=0 exact_recv_per_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131314 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131051 expected_overflow=0 exact_recv_per_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=81920 expected_recv=130919 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=81920 expected_recv=130860 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131059 expected_overflow=0 exact_recv_per_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131051 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131439 expected_overflow=0 exact_recv_per_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131059 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131341 expected_overflow=0 exact_recv_per_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131439 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131341 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=81920 expected_recv=130580 expected_overflow=0 exact_recv_per_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131104 expected_overflow=0 exact_recv_per_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=81920 expected_recv=130580 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=81920 expected_recv=130756 expected_overflow=0 exact_recv_per_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131104 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131180 expected_overflow=0 exact_recv_per_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=81920 expected_recv=130756 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131164 expected_overflow=0 exact_recv_per_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131171 expected_overflow=0 exact_recv_per_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131180 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131359 expected_overflow=0 exact_recv_per_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=81920 expected_recv=131164 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131171 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=81920 expected_recv=130936 expected_overflow=0 exact_recv_per_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131359 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131267 expected_overflow=0 exact_recv_per_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=81920 expected_recv=130936 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131267 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131482 expected_overflow=0 exact_recv_per_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131265 expected_overflow=0 exact_recv_per_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=81920 expected_recv=130439 expected_overflow=0 exact_recv_per_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131482 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131265 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131073 expected_overflow=0 exact_recv_per_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=81920 expected_recv=130439 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131013 expected_overflow=0 exact_recv_per_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131073 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=81920 expected_recv=130471 expected_overflow=0 exact_recv_per_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=81920 expected_recv=131013 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=81920 expected_recv=130471 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131152 expected_overflow=0 exact_recv_per_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131103 expected_overflow=0 exact_recv_per_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131152 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=81920 expected_recv=130934 expected_overflow=0 exact_recv_per_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131103 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131388 expected_overflow=0 exact_recv_per_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=81920 expected_recv=130934 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131388 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131166 expected_overflow=0 exact_recv_per_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=81920 expected_recv=131166 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131190 expected_overflow=0 exact_recv_per_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=81920 expected_recv=131190 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[65523, 65470, 65867, 65395, 65164, 65589, 65954, 65326] avg=65536.0000 max=65954 max_rank=6 min=65164 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[65474, 66126, 65644, 65379, 65324, 65220, 65606, 65515] avg=65536.0000 max=66126 max_rank=1 min=65220 min_rank=5 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65673, 65650, 65571, 65426, 65442, 65582, 65569, 65375] avg=65536.0000 max=65673 max_rank=0 min=65375 min_rank=7 imbalance_ratio=1.002090 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65863, 65482, 65551, 65627, 65186, 65570, 65472, 65537] avg=65536.0000 max=65863 max_rank=0 min=65186 min_rank=4 imbalance_ratio=1.004990 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[65809, 65271, 65611, 65441, 65714, 65701, 65097, 65644] avg=65536.0000 max=65809 max_rank=0 min=65097 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[65670, 65839, 65376, 65762, 65332, 65404, 65405, 65500] avg=65536.0000 max=65839 max_rank=1 min=65332 min_rank=4 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[65620, 65820, 65395, 65396, 65573, 65567, 65216, 65701] avg=65536.0000 max=65820 max_rank=1 min=65216 min_rank=6 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[65650, 65576, 65427, 65678, 65137, 65573, 65414, 65833] avg=65536.0000 max=65833 max_rank=7 min=65137 min_rank=4 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[65226, 65717, 65630, 65380, 65211, 65428, 65885, 65811] avg=65536.0000 max=65885 max_rank=6 min=65211 min_rank=4 imbalance_ratio=1.005325 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[65895, 65238, 65706, 65328, 65741, 65609, 65491, 65280] avg=65536.0000 max=65895 max_rank=0 min=65238 min_rank=1 imbalance_ratio=1.005478 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[65585, 65474, 65726, 65337, 65644, 65826, 65238, 65458] avg=65536.0000 max=65826 max_rank=5 min=65238 min_rank=6 imbalance_ratio=1.004425 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[65532, 65608, 65549, 65392, 65329, 65719, 65642, 65517] avg=65536.0000 max=65719 max_rank=5 min=65329 min_rank=4 imbalance_ratio=1.002792 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[65350, 65322, 65358, 65805, 65669, 65534, 65388, 65862] avg=65536.0000 max=65862 max_rank=7 min=65322 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[65865, 65420, 65698, 65414, 65568, 65430, 65657, 65236] avg=65536.0000 max=65865 max_rank=0 min=65236 min_rank=7 imbalance_ratio=1.005020 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[65660, 65249, 65697, 65599, 65318, 65618, 65510, 65637] avg=65536.0000 max=65697 max_rank=2 min=65249 min_rank=1 imbalance_ratio=1.002457 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=2098264 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506880 peak_demand=66126 required_capacity=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[65577, 65389, 65633, 65344, 65737, 65412, 65511, 65685] avg=65536.0000 max=65737 max_rank=4 min=65344 min_rank=3 imbalance_ratio=1.003067 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=2096883 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508288 peak_demand=66009 required_capacity=1.007217 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=2096073 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508544 peak_demand=65862 required_capacity=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=2098415 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=505728 peak_demand=65895 required_capacity=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=2096082 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509696 peak_demand=65987 required_capacity=1.006882 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=2098095 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506880 peak_demand=65954 required_capacity=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=2095535 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508800 peak_demand=65883 required_capacity=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=2097869 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507008 peak_demand=65954 required_capacity=1.006378 + [2026-08-30 05:16:39] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 473.8 | throughput per GPU (TFLOP/s/GPU): 703.7 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.992991E+00 | loss scale: 1.0 | grad norm: 6.952 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66560 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=81920 expected_recv=130797 expected_overflow=0 exact_recv_per_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=81920 expected_recv=130724 expected_overflow=0 exact_recv_per_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131011 expected_overflow=0 exact_recv_per_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131501 expected_overflow=0 exact_recv_per_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131006 expected_overflow=0 exact_recv_per_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131166 expected_overflow=0 exact_recv_per_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=81920 expected_recv=130797 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=81920 expected_recv=130724 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131072 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131011 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131501 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131299 expected_overflow=0 exact_recv_per_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131006 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131166 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=81920 expected_recv=131299 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=81920 expected_recv=130773 expected_overflow=0 exact_recv_per_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131163 expected_overflow=0 exact_recv_per_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=81920 expected_recv=130757 expected_overflow=0 exact_recv_per_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131120 expected_overflow=0 exact_recv_per_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131536 expected_overflow=0 exact_recv_per_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=81920 expected_recv=130773 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131163 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=81920 expected_recv=130757 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131120 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131007 expected_overflow=0 exact_recv_per_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131288 expected_overflow=0 exact_recv_per_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131536 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131007 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=81920 expected_recv=130932 expected_overflow=0 exact_recv_per_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=81920 expected_recv=131288 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=81920 expected_recv=130932 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131086 expected_overflow=0 exact_recv_per_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=81920 expected_recv=130900 expected_overflow=0 exact_recv_per_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131371 expected_overflow=0 exact_recv_per_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131451 expected_overflow=0 exact_recv_per_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131099 expected_overflow=0 exact_recv_per_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=81920 expected_recv=130404 expected_overflow=0 exact_recv_per_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131086 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=81920 expected_recv=130900 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131371 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131451 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131099 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131412 expected_overflow=0 exact_recv_per_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=81920 expected_recv=130404 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=81920 expected_recv=130853 expected_overflow=0 exact_recv_per_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=81920 expected_recv=131412 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=81920 expected_recv=130853 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131409 expected_overflow=0 exact_recv_per_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130754 expected_overflow=0 exact_recv_per_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131038 expected_overflow=0 exact_recv_per_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131527 expected_overflow=0 exact_recv_per_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131409 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130754 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131258 expected_overflow=0 exact_recv_per_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131527 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130781 expected_overflow=0 exact_recv_per_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131038 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130681 expected_overflow=0 exact_recv_per_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131128 expected_overflow=0 exact_recv_per_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131258 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130781 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=81920 expected_recv=130681 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=81920 expected_recv=131128 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130278 expected_overflow=0 exact_recv_per_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131686 expected_overflow=0 exact_recv_per_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130278 actual_valid_recv=131072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131686 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131072 slack_slots=32768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131757 expected_overflow=0 exact_recv_per_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130977 expected_overflow=0 exact_recv_per_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131757 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130812 expected_overflow=0 exact_recv_per_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130977 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131141 expected_overflow=0 exact_recv_per_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130853 expected_overflow=0 exact_recv_per_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131072 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130812 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=81920 expected_recv=131141 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=81920 expected_recv=130853 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=81920 expected_recv=130993 expected_overflow=0 exact_recv_per_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=81920 expected_recv=130997 expected_overflow=0 exact_recv_per_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=81920 expected_recv=130993 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=81920 expected_recv=130997 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=81920 expected_recv=130857 expected_overflow=0 exact_recv_per_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=81920 expected_recv=131699 expected_overflow=0 exact_recv_per_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=81920 expected_recv=130857 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=81920 expected_recv=130550 expected_overflow=0 exact_recv_per_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=81920 expected_recv=131062 expected_overflow=0 exact_recv_per_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=81920 expected_recv=131699 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=81920 expected_recv=131023 expected_overflow=0 exact_recv_per_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=81920 expected_recv=130550 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=81920 expected_recv=131395 expected_overflow=0 exact_recv_per_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=81920 expected_recv=131062 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131066 expected_overflow=0 exact_recv_per_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=81920 expected_recv=131023 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=81920 expected_recv=131395 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=81920 expected_recv=130963 expected_overflow=0 exact_recv_per_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131066 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=81920 expected_recv=130963 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131263 expected_overflow=0 exact_recv_per_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131151 expected_overflow=0 exact_recv_per_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131263 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=81920 expected_recv=130881 expected_overflow=0 exact_recv_per_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131151 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=81920 expected_recv=130881 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131085 expected_overflow=0 exact_recv_per_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131316 expected_overflow=0 exact_recv_per_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=81920 expected_recv=130851 expected_overflow=0 exact_recv_per_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131085 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=81920 expected_recv=131316 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130851 expected_overflow=0 exact_recv_per_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=81920 expected_recv=130851 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131231 expected_overflow=0 exact_recv_per_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130851 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131231 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130852 expected_overflow=0 exact_recv_per_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130822 expected_overflow=0 exact_recv_per_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130852 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130822 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130915 expected_overflow=0 exact_recv_per_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131086 expected_overflow=0 exact_recv_per_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131736 expected_overflow=0 exact_recv_per_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=81920 expected_recv=130915 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131083 expected_overflow=0 exact_recv_per_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131086 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=81920 expected_recv=130866 expected_overflow=0 exact_recv_per_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131736 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131067 expected_overflow=0 exact_recv_per_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=81920 expected_recv=131083 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=81920 expected_recv=130866 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131321 expected_overflow=0 exact_recv_per_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131067 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=81920 expected_recv=130627 expected_overflow=0 exact_recv_per_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131321 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=81920 expected_recv=130627 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131310 expected_overflow=0 exact_recv_per_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=81920 expected_recv=130868 expected_overflow=0 exact_recv_per_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131310 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=81920 expected_recv=130940 expected_overflow=0 exact_recv_per_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=81920 expected_recv=130490 expected_overflow=0 exact_recv_per_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131577 expected_overflow=0 exact_recv_per_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=81920 expected_recv=130868 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=81920 expected_recv=130940 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=81920 expected_recv=130490 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=81920 expected_recv=131577 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131120 expected_overflow=0 exact_recv_per_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131094 expected_overflow=0 exact_recv_per_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131093 expected_overflow=0 exact_recv_per_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131120 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131094 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131093 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131221 expected_overflow=0 exact_recv_per_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131210 expected_overflow=0 exact_recv_per_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131221 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131455 expected_overflow=0 exact_recv_per_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131210 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=81920 expected_recv=130880 expected_overflow=0 exact_recv_per_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=81920 expected_recv=131455 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=81920 expected_recv=130893 expected_overflow=0 exact_recv_per_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=81920 expected_recv=130880 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=81920 expected_recv=130893 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131463 expected_overflow=0 exact_recv_per_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=81920 expected_recv=130748 expected_overflow=0 exact_recv_per_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131253 expected_overflow=0 exact_recv_per_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131463 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=81920 expected_recv=130748 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131253 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131293 expected_overflow=0 exact_recv_per_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131293 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=81920 expected_recv=130944 expected_overflow=0 exact_recv_per_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131152 expected_overflow=0 exact_recv_per_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=81920 expected_recv=130944 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=81920 expected_recv=130602 expected_overflow=0 exact_recv_per_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=81920 expected_recv=131152 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=81920 expected_recv=130843 expected_overflow=0 exact_recv_per_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=81920 expected_recv=130602 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=81920 expected_recv=130843 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131175 expected_overflow=0 exact_recv_per_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131055 expected_overflow=0 exact_recv_per_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=81920 expected_recv=130733 expected_overflow=0 exact_recv_per_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131175 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131055 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=81920 expected_recv=130733 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131316 expected_overflow=0 exact_recv_per_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131316 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131674 expected_overflow=0 exact_recv_per_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131322 expected_overflow=0 exact_recv_per_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131674 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=81920 expected_recv=130761 expected_overflow=0 exact_recv_per_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=81920 expected_recv=131322 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=81920 expected_recv=130699 expected_overflow=0 exact_recv_per_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=81920 expected_recv=130761 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=81920 expected_recv=130908 expected_overflow=0 exact_recv_per_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=81920 expected_recv=130699 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131410 expected_overflow=0 exact_recv_per_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=81920 expected_recv=130793 expected_overflow=0 exact_recv_per_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=81920 expected_recv=130908 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131410 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=81920 expected_recv=130793 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131445 expected_overflow=0 exact_recv_per_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131445 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131121 expected_overflow=0 exact_recv_per_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131280 expected_overflow=0 exact_recv_per_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131121 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=81920 expected_recv=131106 expected_overflow=0 exact_recv_per_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=81920 expected_recv=131280 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=81920 expected_recv=131106 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=81920 expected_recv=130858 expected_overflow=0 exact_recv_per_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=81920 expected_recv=131092 expected_overflow=0 exact_recv_per_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=81920 expected_recv=130891 expected_overflow=0 exact_recv_per_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=81920 expected_recv=130858 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=81920 expected_recv=131061 expected_overflow=0 exact_recv_per_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=81920 expected_recv=131092 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=81920 expected_recv=130891 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=81920 expected_recv=130948 expected_overflow=0 exact_recv_per_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=81920 expected_recv=131061 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=81920 expected_recv=130948 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=81920 expected_recv=130947 expected_overflow=0 exact_recv_per_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=81920 expected_recv=131257 expected_overflow=0 exact_recv_per_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=81920 expected_recv=130947 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=81920 expected_recv=130814 expected_overflow=0 exact_recv_per_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=81920 expected_recv=131257 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=81920 expected_recv=130814 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=81920 expected_recv=131274 expected_overflow=0 exact_recv_per_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131323 expected_overflow=0 exact_recv_per_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131285 expected_overflow=0 exact_recv_per_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=81920 expected_recv=131274 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131134 expected_overflow=0 exact_recv_per_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131323 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131285 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=81920 expected_recv=130611 expected_overflow=0 exact_recv_per_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131134 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=81920 expected_recv=130611 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131232 expected_overflow=0 exact_recv_per_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131435 expected_overflow=0 exact_recv_per_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131632 expected_overflow=0 exact_recv_per_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131232 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=81920 expected_recv=130742 expected_overflow=0 exact_recv_per_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=81920 expected_recv=130533 expected_overflow=0 exact_recv_per_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=81920 expected_recv=130716 expected_overflow=0 exact_recv_per_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=81920 expected_recv=131435 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131632 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=81920 expected_recv=130742 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=81920 expected_recv=130533 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=81920 expected_recv=130716 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=81920 expected_recv=130504 expected_overflow=0 exact_recv_per_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=81920 expected_recv=130504 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131814 expected_overflow=0 exact_recv_per_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131187 expected_overflow=0 exact_recv_per_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131814 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131239 expected_overflow=0 exact_recv_per_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131187 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=81920 expected_recv=130951 expected_overflow=0 exact_recv_per_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=81920 expected_recv=131239 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=81920 expected_recv=130951 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[65209, 65177, 65446, 66030, 65854, 65635, 65640, 65297] avg=65536.0000 max=66030 max_rank=3 min=65177 min_rank=1 imbalance_ratio=1.007538 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[65580, 65350, 65449, 65768, 65448, 65690, 65481, 65522] avg=65536.0000 max=65768 max_rank=3 min=65350 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[65366, 65628, 65631, 65292, 65249, 65930, 65799, 65393] avg=65536.0000 max=65930 max_rank=5 min=65249 min_rank=4 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[65336, 65809, 65736, 65411, 65504, 65776, 65327, 65389] avg=65536.0000 max=65809 max_rank=1 min=65327 min_rank=6 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[65324, 65488, 65859, 65494, 65399, 65741, 65436, 65547] avg=65536.0000 max=65859 max_rank=2 min=65324 min_rank=0 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65395, 65755, 65415, 65448, 65877, 65312, 65569, 65517] avg=65536.0000 max=65877 max_rank=4 min=65312 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65556, 65540, 65380, 65361, 65497, 65852, 65491, 65611] avg=65536.0000 max=65852 max_rank=5 min=65361 min_rank=3 imbalance_ratio=1.004822 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[65294, 65421, 65555, 65649, 65468, 65499, 65783, 65619] avg=65536.0000 max=65783 max_rank=6 min=65294 min_rank=0 imbalance_ratio=1.003769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[65143, 65379, 65471, 65597, 66026, 65460, 65666, 65546] avg=65536.0000 max=66026 max_rank=4 min=65143 min_rank=0 imbalance_ratio=1.007477 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[65229, 65532, 65455, 65568, 65615, 65494, 65724, 65671] avg=65536.0000 max=65724 max_rank=6 min=65229 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[65279, 65771, 65809, 65713, 65363, 65278, 65806, 65269] avg=65536.0000 max=65809 max_rank=2 min=65269 min_rank=7 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[65281, 65246, 65633, 65589, 65663, 65496, 65739, 65641] avg=65536.0000 max=65739 max_rank=6 min=65246 min_rank=1 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[65495, 65505, 65329, 65891, 65296, 65493, 65688, 65591] avg=65536.0000 max=65891 max_rank=3 min=65296 min_rank=4 imbalance_ratio=1.005417 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[65667, 65580, 65327, 65655, 65682, 65402, 65470, 65505] avg=65536.0000 max=65682 max_rank=4 min=65327 min_rank=2 imbalance_ratio=1.002228 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[65435, 65693, 65710, 65365, 65435, 65653, 65598, 65399] avg=65536.0000 max=65710 max_rank=2 min=65365 min_rank=3 imbalance_ratio=1.002655 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[65848, 65205, 65390, 65862, 65415, 65344, 65593, 65631] avg=65536.0000 max=65862 max_rank=3 min=65205 min_rank=1 imbalance_ratio=1.004974 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=2094228 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509952 peak_demand=65863 required_capacity=1.004990 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=2096426 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508032 peak_demand=66033 required_capacity=1.007584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=2097806 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507520 peak_demand=65859 required_capacity=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=2098175 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506752 peak_demand=66016 required_capacity=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=2095737 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509312 peak_demand=65944 required_capacity=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=2096231 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508416 peak_demand=66026 required_capacity=1.007477 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=2098770 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507008 peak_demand=66030 required_capacity=1.007538 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=2099843 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=504832 peak_demand=65953 required_capacity=1.006363 + [2026-08-30 05:16:40] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 472.0 | throughput per GPU (TFLOP/s/GPU): 706.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.490677E+00 | loss scale: 1.0 | grad norm: 5.586 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66560 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131160 expected_overflow=0 exact_recv_per_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=81920 expected_recv=130869 expected_overflow=0 exact_recv_per_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131148 expected_overflow=0 exact_recv_per_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131210 expected_overflow=0 exact_recv_per_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=81920 expected_recv=130611 expected_overflow=0 exact_recv_per_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131007 expected_overflow=0 exact_recv_per_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131160 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=81920 expected_recv=130869 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131148 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131229 expected_overflow=0 exact_recv_per_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131210 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=81920 expected_recv=130611 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131342 expected_overflow=0 exact_recv_per_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131007 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131229 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=81920 expected_recv=131342 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131084 expected_overflow=0 exact_recv_per_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131319 expected_overflow=0 exact_recv_per_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131533 expected_overflow=0 exact_recv_per_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131084 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131319 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131533 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=81920 expected_recv=130445 expected_overflow=0 exact_recv_per_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131038 expected_overflow=0 exact_recv_per_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=81920 expected_recv=130927 expected_overflow=0 exact_recv_per_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=81920 expected_recv=130445 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131038 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=81920 expected_recv=130988 expected_overflow=0 exact_recv_per_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=81920 expected_recv=130927 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131242 expected_overflow=0 exact_recv_per_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=81920 expected_recv=130988 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131469 expected_overflow=0 exact_recv_per_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131523 expected_overflow=0 exact_recv_per_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=81920 expected_recv=131242 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131469 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131020 expected_overflow=0 exact_recv_per_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131523 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131020 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=81920 expected_recv=130772 expected_overflow=0 exact_recv_per_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=81920 expected_recv=130817 expected_overflow=0 exact_recv_per_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=81920 expected_recv=130441 expected_overflow=0 exact_recv_per_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131320 expected_overflow=0 exact_recv_per_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=81920 expected_recv=130772 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=81920 expected_recv=130817 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=81920 expected_recv=130441 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130810 expected_overflow=0 exact_recv_per_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131320 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131214 expected_overflow=0 exact_recv_per_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131044 expected_overflow=0 exact_recv_per_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130810 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=81920 expected_recv=131214 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131044 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131112 expected_overflow=0 exact_recv_per_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131112 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131069 expected_overflow=0 exact_recv_per_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131363 expected_overflow=0 exact_recv_per_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131420 expected_overflow=0 exact_recv_per_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130801 expected_overflow=0 exact_recv_per_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131069 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=81920 expected_recv=130664 expected_overflow=0 exact_recv_per_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131363 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=81920 expected_recv=131420 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130801 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130957 expected_overflow=0 exact_recv_per_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=81920 expected_recv=130664 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=81920 expected_recv=130536 expected_overflow=0 exact_recv_per_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=81920 expected_recv=130957 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131210 expected_overflow=0 exact_recv_per_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=81920 expected_recv=130536 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131210 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131096 expected_overflow=0 exact_recv_per_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131127 expected_overflow=0 exact_recv_per_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131158 expected_overflow=0 exact_recv_per_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131096 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131127 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131049 expected_overflow=0 exact_recv_per_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131158 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131225 expected_overflow=0 exact_recv_per_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131049 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131736 expected_overflow=0 exact_recv_per_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131207 expected_overflow=0 exact_recv_per_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131225 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=81920 expected_recv=131736 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131207 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=81920 expected_recv=130644 expected_overflow=0 exact_recv_per_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=81920 expected_recv=130644 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131256 expected_overflow=0 exact_recv_per_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131286 expected_overflow=0 exact_recv_per_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131433 expected_overflow=0 exact_recv_per_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131256 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131286 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=81920 expected_recv=130386 expected_overflow=0 exact_recv_per_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131433 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131024 expected_overflow=0 exact_recv_per_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=81920 expected_recv=130386 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131139 expected_overflow=0 exact_recv_per_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=81920 expected_recv=130959 expected_overflow=0 exact_recv_per_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131024 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=81920 expected_recv=131139 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=81920 expected_recv=130959 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131225 expected_overflow=0 exact_recv_per_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131225 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=81920 expected_recv=130851 expected_overflow=0 exact_recv_per_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=81920 expected_recv=130879 expected_overflow=0 exact_recv_per_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=81920 expected_recv=130889 expected_overflow=0 exact_recv_per_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=81920 expected_recv=130851 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=81920 expected_recv=130879 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131042 expected_overflow=0 exact_recv_per_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=81920 expected_recv=130889 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131453 expected_overflow=0 exact_recv_per_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131042 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131707 expected_overflow=0 exact_recv_per_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131593 expected_overflow=0 exact_recv_per_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131453 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=81920 expected_recv=131707 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131593 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=81920 expected_recv=130903 expected_overflow=0 exact_recv_per_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=81920 expected_recv=130903 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=81920 expected_recv=130578 expected_overflow=0 exact_recv_per_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131168 expected_overflow=0 exact_recv_per_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131125 expected_overflow=0 exact_recv_per_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=81920 expected_recv=130578 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131185 expected_overflow=0 exact_recv_per_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131168 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131125 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=81920 expected_recv=130571 expected_overflow=0 exact_recv_per_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131036 expected_overflow=0 exact_recv_per_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=81920 expected_recv=131185 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=81920 expected_recv=130713 expected_overflow=0 exact_recv_per_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=81920 expected_recv=130571 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131036 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=81920 expected_recv=130713 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=81920 expected_recv=130719 expected_overflow=0 exact_recv_per_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=81920 expected_recv=130719 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131482 expected_overflow=0 exact_recv_per_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131359 expected_overflow=0 exact_recv_per_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131531 expected_overflow=0 exact_recv_per_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131482 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=81920 expected_recv=130672 expected_overflow=0 exact_recv_per_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131359 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131531 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131064 expected_overflow=0 exact_recv_per_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=81920 expected_recv=130632 expected_overflow=0 exact_recv_per_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=81920 expected_recv=130672 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=81920 expected_recv=130879 expected_overflow=0 exact_recv_per_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=81920 expected_recv=131064 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=81920 expected_recv=130632 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=81920 expected_recv=130946 expected_overflow=0 exact_recv_per_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=81920 expected_recv=130879 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=81920 expected_recv=130946 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=81920 expected_recv=130882 expected_overflow=0 exact_recv_per_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131616 expected_overflow=0 exact_recv_per_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131329 expected_overflow=0 exact_recv_per_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=81920 expected_recv=130882 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=81920 expected_recv=130690 expected_overflow=0 exact_recv_per_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131616 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131602 expected_overflow=0 exact_recv_per_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=81920 expected_recv=131131 expected_overflow=0 exact_recv_per_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131329 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=81920 expected_recv=131325 expected_overflow=0 exact_recv_per_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=81920 expected_recv=130690 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=81920 expected_recv=131602 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=81920 expected_recv=131131 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=81920 expected_recv=130816 expected_overflow=0 exact_recv_per_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=81920 expected_recv=131325 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=81920 expected_recv=130816 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=81920 expected_recv=130828 expected_overflow=0 exact_recv_per_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=81920 expected_recv=130859 expected_overflow=0 exact_recv_per_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=81920 expected_recv=130585 expected_overflow=0 exact_recv_per_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=81920 expected_recv=130828 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=81920 expected_recv=131879 expected_overflow=0 exact_recv_per_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=81920 expected_recv=130859 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=81920 expected_recv=131153 expected_overflow=0 exact_recv_per_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131318 expected_overflow=0 exact_recv_per_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=81920 expected_recv=130585 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=81920 expected_recv=131879 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130932 expected_overflow=0 exact_recv_per_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=81920 expected_recv=131153 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131318 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130637 expected_overflow=0 exact_recv_per_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130932 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130637 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131342 expected_overflow=0 exact_recv_per_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131378 expected_overflow=0 exact_recv_per_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131618 expected_overflow=0 exact_recv_per_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131342 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131378 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=81920 expected_recv=131618 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130473 expected_overflow=0 exact_recv_per_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130878 expected_overflow=0 exact_recv_per_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=81920 expected_recv=130778 expected_overflow=0 exact_recv_per_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130473 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131058 expected_overflow=0 exact_recv_per_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=81920 expected_recv=130878 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131171 expected_overflow=0 exact_recv_per_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=81920 expected_recv=130778 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131058 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131171 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=81920 expected_recv=130850 expected_overflow=0 exact_recv_per_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131512 expected_overflow=0 exact_recv_per_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=81920 expected_recv=130456 expected_overflow=0 exact_recv_per_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=81920 expected_recv=130850 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131512 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=81920 expected_recv=130456 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131566 expected_overflow=0 exact_recv_per_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131185 expected_overflow=0 exact_recv_per_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=81920 expected_recv=130603 expected_overflow=0 exact_recv_per_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=81920 expected_recv=130974 expected_overflow=0 exact_recv_per_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131566 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=81920 expected_recv=131185 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=81920 expected_recv=130603 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131411 expected_overflow=0 exact_recv_per_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=81920 expected_recv=130974 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131411 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=81920 expected_recv=130711 expected_overflow=0 exact_recv_per_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131197 expected_overflow=0 exact_recv_per_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=81920 expected_recv=130775 expected_overflow=0 exact_recv_per_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=81920 expected_recv=130711 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131197 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=81920 expected_recv=130775 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131761 expected_overflow=0 exact_recv_per_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131525 expected_overflow=0 exact_recv_per_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131144 expected_overflow=0 exact_recv_per_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130792 expected_overflow=0 exact_recv_per_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131761 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131525 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130873 expected_overflow=0 exact_recv_per_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=81920 expected_recv=131144 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130792 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130873 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130773 expected_overflow=0 exact_recv_per_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131084 expected_overflow=0 exact_recv_per_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130775 expected_overflow=0 exact_recv_per_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130773 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131125 expected_overflow=0 exact_recv_per_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131084 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=81920 expected_recv=130775 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=81920 expected_recv=130846 expected_overflow=0 exact_recv_per_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131629 expected_overflow=0 exact_recv_per_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131343 expected_overflow=0 exact_recv_per_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131125 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=81920 expected_recv=130846 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=81920 expected_recv=131629 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131343 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131329 expected_overflow=0 exact_recv_per_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131329 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131042 expected_overflow=0 exact_recv_per_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131148 expected_overflow=0 exact_recv_per_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131042 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131042 expected_overflow=0 exact_recv_per_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131148 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131000 expected_overflow=0 exact_recv_per_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131042 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=81920 expected_recv=131000 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=81920 expected_recv=130826 expected_overflow=0 exact_recv_per_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=81920 expected_recv=130826 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[65391, 65573, 65289, 65681, 65812, 65725, 65502, 65315] avg=65536.0000 max=65812 max_rank=4 min=65289 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[65520, 65421, 66018, 65548, 65541, 65745, 65439, 65056] avg=65536.0000 max=66018 max_rank=2 min=65056 min_rank=7 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[65442, 65412, 65613, 65419, 65758, 65613, 65454, 65577] avg=65536.0000 max=65758 max_rank=4 min=65412 min_rank=1 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[65109, 65683, 65686, 65529, 65687, 65548, 65511, 65535] avg=65536.0000 max=65687 max_rank=4 min=65109 min_rank=0 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[65216, 65816, 65472, 65751, 65444, 65255, 65610, 65724] avg=65536.0000 max=65816 max_rank=1 min=65216 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[65558, 65615, 65324, 65718, 65022, 65734, 65543, 65774] avg=65536.0000 max=65774 max_rank=7 min=65022 min_rank=4 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65497, 65946, 65502, 65395, 65707, 65420, 65280, 65541] avg=65536.0000 max=65946 max_rank=1 min=65280 min_rank=6 imbalance_ratio=1.006256 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[65621, 64873, 65257, 65555, 65725, 65837, 65602, 65818] avg=65536.0000 max=65837 max_rank=5 min=64873 min_rank=1 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[65183, 65502, 65465, 65837, 65312, 65560, 65808, 65621] avg=65536.0000 max=65837 max_rank=3 min=65183 min_rank=0 imbalance_ratio=1.004593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[65378, 65927, 65242, 65515, 65501, 65524, 65419, 65782] avg=65536.0000 max=65927 max_rank=1 min=65242 min_rank=2 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[65481, 65522, 65364, 65586, 66046, 65648, 65206, 65435] avg=65536.0000 max=66046 max_rank=4 min=65206 min_rank=6 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[65717, 65453, 65719, 65318, 65320, 65313, 65663, 65785] avg=65536.0000 max=65785 max_rank=7 min=65313 min_rank=5 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[65585, 65746, 65639, 65427, 65743, 65309, 65025, 65814] avg=65536.0000 max=65814 max_rank=7 min=65025 min_rank=6 imbalance_ratio=1.004242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[65080, 65675, 65636, 65635, 65843, 65497, 65293, 65629] avg=65536.0000 max=65843 max_rank=4 min=65080 min_rank=0 imbalance_ratio=1.004684 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[65790, 65780, 65397, 65411, 65324, 65513, 65492, 65581] avg=65536.0000 max=65790 max_rank=0 min=65324 min_rank=4 imbalance_ratio=1.003876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[65159, 65605, 65530, 65723, 65693, 65620, 65439, 65519] avg=65536.0000 max=65723 max_rank=3 min=65159 min_rank=0 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=2097345 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508032 peak_demand=65910 required_capacity=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=2096758 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507904 peak_demand=66027 required_capacity=1.007492 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=2096901 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507520 peak_demand=65834 required_capacity=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=2096926 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507648 peak_demand=65955 required_capacity=1.006393 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=2099389 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506112 peak_demand=65946 required_capacity=1.006256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=2096418 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509056 peak_demand=66018 required_capacity=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=2097166 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508672 peak_demand=66046 required_capacity=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=2096313 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508288 peak_demand=66024 required_capacity=1.007446 + [2026-08-30 05:16:40] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 471.7 | throughput per GPU (TFLOP/s/GPU): 706.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.257784E+00 | loss scale: 1.0 | grad norm: 5.505 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66560 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131397 expected_overflow=0 exact_recv_per_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=81920 expected_recv=130849 expected_overflow=0 exact_recv_per_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131151 expected_overflow=0 exact_recv_per_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131273 expected_overflow=0 exact_recv_per_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131013 expected_overflow=0 exact_recv_per_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=81920 expected_recv=130936 expected_overflow=0 exact_recv_per_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131164 expected_overflow=0 exact_recv_per_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131397 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=81920 expected_recv=130849 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=81920 expected_recv=130793 expected_overflow=0 exact_recv_per_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131151 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131273 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131013 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=81920 expected_recv=130936 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=81920 expected_recv=131164 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=81920 expected_recv=130793 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=81920 expected_recv=130810 expected_overflow=0 exact_recv_per_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131232 expected_overflow=0 exact_recv_per_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131461 expected_overflow=0 exact_recv_per_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=81920 expected_recv=130810 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131232 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=81920 expected_recv=130868 expected_overflow=0 exact_recv_per_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131033 expected_overflow=0 exact_recv_per_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131461 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131013 expected_overflow=0 exact_recv_per_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131140 expected_overflow=0 exact_recv_per_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=81920 expected_recv=130868 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131033 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131019 expected_overflow=0 exact_recv_per_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131140 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131013 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=81920 expected_recv=131019 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=81920 expected_recv=130736 expected_overflow=0 exact_recv_per_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=81920 expected_recv=130959 expected_overflow=0 exact_recv_per_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=81920 expected_recv=130736 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=81920 expected_recv=130959 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131195 expected_overflow=0 exact_recv_per_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131118 expected_overflow=0 exact_recv_per_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=81920 expected_recv=130720 expected_overflow=0 exact_recv_per_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131294 expected_overflow=0 exact_recv_per_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131195 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131118 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=81920 expected_recv=130720 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131349 expected_overflow=0 exact_recv_per_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131294 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131349 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131205 expected_overflow=0 exact_recv_per_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130934 expected_overflow=0 exact_recv_per_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=81920 expected_recv=131180 expected_overflow=0 exact_recv_per_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130934 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=81920 expected_recv=131205 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=81920 expected_recv=131180 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130579 expected_overflow=0 exact_recv_per_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130579 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=81920 expected_recv=131753 expected_overflow=0 exact_recv_per_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130749 expected_overflow=0 exact_recv_per_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130991 expected_overflow=0 exact_recv_per_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=81920 expected_recv=131753 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130749 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=81920 expected_recv=130991 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=81920 expected_recv=131195 expected_overflow=0 exact_recv_per_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=81920 expected_recv=130864 expected_overflow=0 exact_recv_per_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=81920 expected_recv=131195 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=81920 expected_recv=131195 expected_overflow=0 exact_recv_per_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=81920 expected_recv=130749 expected_overflow=0 exact_recv_per_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=81920 expected_recv=130864 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=81920 expected_recv=131195 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=81920 expected_recv=130749 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131411 expected_overflow=0 exact_recv_per_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131092 expected_overflow=0 exact_recv_per_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131108 expected_overflow=0 exact_recv_per_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131563 expected_overflow=0 exact_recv_per_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131411 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131092 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131108 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131207 expected_overflow=0 exact_recv_per_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131563 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=81920 expected_recv=130966 expected_overflow=0 exact_recv_per_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=81920 expected_recv=131207 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=81920 expected_recv=130582 expected_overflow=0 exact_recv_per_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=81920 expected_recv=130598 expected_overflow=0 exact_recv_per_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=81920 expected_recv=130966 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=81920 expected_recv=130582 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=81920 expected_recv=130598 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131324 expected_overflow=0 exact_recv_per_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131090 expected_overflow=0 exact_recv_per_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131346 expected_overflow=0 exact_recv_per_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131324 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131090 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131025 expected_overflow=0 exact_recv_per_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131282 expected_overflow=0 exact_recv_per_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131346 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131025 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=81920 expected_recv=131282 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=81920 expected_recv=130849 expected_overflow=0 exact_recv_per_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=81920 expected_recv=131649 expected_overflow=0 exact_recv_per_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=81920 expected_recv=130945 expected_overflow=0 exact_recv_per_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=81920 expected_recv=130849 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=81920 expected_recv=131649 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=81920 expected_recv=130945 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=81920 expected_recv=130407 expected_overflow=0 exact_recv_per_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=81920 expected_recv=130957 expected_overflow=0 exact_recv_per_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=81920 expected_recv=131029 expected_overflow=0 exact_recv_per_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=81920 expected_recv=130407 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=81920 expected_recv=130957 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=81920 expected_recv=131029 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=81920 expected_recv=131414 expected_overflow=0 exact_recv_per_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=81920 expected_recv=131448 expected_overflow=0 exact_recv_per_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=81920 expected_recv=131414 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=81920 expected_recv=131448 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131365 expected_overflow=0 exact_recv_per_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131176 expected_overflow=0 exact_recv_per_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131365 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=81920 expected_recv=130823 expected_overflow=0 exact_recv_per_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131176 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131170 expected_overflow=0 exact_recv_per_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=81920 expected_recv=130823 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=81920 expected_recv=130802 expected_overflow=0 exact_recv_per_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=81920 expected_recv=130641 expected_overflow=0 exact_recv_per_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131170 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=81920 expected_recv=130802 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=81920 expected_recv=130641 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131775 expected_overflow=0 exact_recv_per_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131016 expected_overflow=0 exact_recv_per_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131775 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=81920 expected_recv=131016 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131278 expected_overflow=0 exact_recv_per_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131439 expected_overflow=0 exact_recv_per_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131278 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=81920 expected_recv=130631 expected_overflow=0 exact_recv_per_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131439 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131399 expected_overflow=0 exact_recv_per_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=81920 expected_recv=130631 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131402 expected_overflow=0 exact_recv_per_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=81920 expected_recv=130465 expected_overflow=0 exact_recv_per_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131399 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=81920 expected_recv=130905 expected_overflow=0 exact_recv_per_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131402 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=81920 expected_recv=130328 expected_overflow=0 exact_recv_per_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=81920 expected_recv=130465 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=81920 expected_recv=130905 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131309 expected_overflow=0 exact_recv_per_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=81920 expected_recv=130328 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131309 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=81920 expected_recv=130577 expected_overflow=0 exact_recv_per_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131360 expected_overflow=0 exact_recv_per_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=81920 expected_recv=130577 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131634 expected_overflow=0 exact_recv_per_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=81920 expected_recv=131360 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131294 expected_overflow=0 exact_recv_per_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131634 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=81920 expected_recv=130545 expected_overflow=0 exact_recv_per_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=81920 expected_recv=130849 expected_overflow=0 exact_recv_per_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131294 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131228 expected_overflow=0 exact_recv_per_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=81920 expected_recv=130545 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=81920 expected_recv=130849 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=81920 expected_recv=130900 expected_overflow=0 exact_recv_per_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131228 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=81920 expected_recv=130900 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131087 expected_overflow=0 exact_recv_per_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131087 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131140 expected_overflow=0 exact_recv_per_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131126 expected_overflow=0 exact_recv_per_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131072 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=81920 expected_recv=131140 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131298 expected_overflow=0 exact_recv_per_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131126 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=81920 expected_recv=130758 expected_overflow=0 exact_recv_per_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131215 expected_overflow=0 exact_recv_per_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131298 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=81920 expected_recv=131405 expected_overflow=0 exact_recv_per_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=81920 expected_recv=130758 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131215 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=81920 expected_recv=130910 expected_overflow=0 exact_recv_per_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=81920 expected_recv=131405 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=81920 expected_recv=130910 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=81920 expected_recv=130924 expected_overflow=0 exact_recv_per_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131120 expected_overflow=0 exact_recv_per_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=81920 expected_recv=130924 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=81920 expected_recv=130857 expected_overflow=0 exact_recv_per_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=81920 expected_recv=131120 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=81920 expected_recv=131162 expected_overflow=0 exact_recv_per_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=81920 expected_recv=130589 expected_overflow=0 exact_recv_per_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=81920 expected_recv=130857 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=81920 expected_recv=131513 expected_overflow=0 exact_recv_per_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=81920 expected_recv=130924 expected_overflow=0 exact_recv_per_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131079 expected_overflow=0 exact_recv_per_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=81920 expected_recv=131162 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=81920 expected_recv=130589 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=81920 expected_recv=131513 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=81920 expected_recv=130924 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131079 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131390 expected_overflow=0 exact_recv_per_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131390 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=81920 expected_recv=131216 expected_overflow=0 exact_recv_per_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=81920 expected_recv=130843 expected_overflow=0 exact_recv_per_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=81920 expected_recv=131216 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=81920 expected_recv=130666 expected_overflow=0 exact_recv_per_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131092 expected_overflow=0 exact_recv_per_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=81920 expected_recv=130843 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=81920 expected_recv=130983 expected_overflow=0 exact_recv_per_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=81920 expected_recv=130899 expected_overflow=0 exact_recv_per_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131683 expected_overflow=0 exact_recv_per_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=81920 expected_recv=130666 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131092 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=81920 expected_recv=130983 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=81920 expected_recv=130899 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131683 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131115 expected_overflow=0 exact_recv_per_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131115 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131599 expected_overflow=0 exact_recv_per_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=81920 expected_recv=130641 expected_overflow=0 exact_recv_per_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=81920 expected_recv=131599 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131022 expected_overflow=0 exact_recv_per_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=81920 expected_recv=130641 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131083 expected_overflow=0 exact_recv_per_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131022 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131353 expected_overflow=0 exact_recv_per_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=81920 expected_recv=130215 expected_overflow=0 exact_recv_per_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131044 expected_overflow=0 exact_recv_per_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131083 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131353 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=81920 expected_recv=130215 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131044 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=81920 expected_recv=130852 expected_overflow=0 exact_recv_per_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=81920 expected_recv=130852 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131089 expected_overflow=0 exact_recv_per_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=81920 expected_recv=131089 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131686 expected_overflow=0 exact_recv_per_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=81920 expected_recv=130910 expected_overflow=0 exact_recv_per_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131686 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131143 expected_overflow=0 exact_recv_per_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131523 expected_overflow=0 exact_recv_per_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=81920 expected_recv=130910 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131101 expected_overflow=0 exact_recv_per_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131052 expected_overflow=0 exact_recv_per_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131143 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131050 expected_overflow=0 exact_recv_per_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131523 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131101 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131052 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131050 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131365 expected_overflow=0 exact_recv_per_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=81920 expected_recv=131365 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=81920 expected_recv=130834 expected_overflow=0 exact_recv_per_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131251 expected_overflow=0 exact_recv_per_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131383 expected_overflow=0 exact_recv_per_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=81920 expected_recv=130834 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131251 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131383 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=81920 expected_recv=130423 expected_overflow=0 exact_recv_per_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=81920 expected_recv=130423 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131011 expected_overflow=0 exact_recv_per_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=81920 expected_recv=131011 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[65504, 65741, 65565, 65442, 65549, 65383, 65567, 65537] avg=65536.0000 max=65741 max_rank=1 min=65383 min_rank=5 imbalance_ratio=1.003128 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[65426, 65269, 65896, 65481, 65324, 65829, 65631, 65432] avg=65536.0000 max=65896 max_rank=2 min=65269 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[65470, 65498, 65568, 65323, 65664, 65706, 65541, 65518] avg=65536.0000 max=65706 max_rank=5 min=65323 min_rank=3 imbalance_ratio=1.002594 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[65192, 66006, 65406, 65297, 65655, 66149, 65191, 65392] avg=65536.0000 max=66149 max_rank=5 min=65191 min_rank=6 imbalance_ratio=1.009354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[65284, 65453, 65397, 65215, 66036, 65749, 65629, 65525] avg=65536.0000 max=66036 max_rank=4 min=65215 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[65306, 65499, 65598, 65404, 65464, 65735, 65717, 65565] avg=65536.0000 max=65735 max_rank=5 min=65306 min_rank=0 imbalance_ratio=1.003036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65304, 65830, 65348, 65532, 65984, 65219, 65335, 65736] avg=65536.0000 max=65984 max_rank=4 min=65219 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[65974, 65540, 65463, 65483, 66013, 65409, 65153, 65253] avg=65536.0000 max=66013 max_rank=4 min=65153 min_rank=6 imbalance_ratio=1.007278 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[65246, 65872, 65797, 66036, 65643, 65751, 64992, 64951] avg=65536.0000 max=66036 max_rank=3 min=64951 min_rank=7 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[65860, 65380, 65880, 65417, 65287, 65706, 65320, 65438] avg=65536.0000 max=65880 max_rank=2 min=65287 min_rank=4 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[65331, 65468, 65699, 65446, 65701, 65478, 65438, 65727] avg=65536.0000 max=65727 max_rank=7 min=65331 min_rank=0 imbalance_ratio=1.002914 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[65651, 65545, 65317, 65665, 65583, 65471, 65669, 65387] avg=65536.0000 max=65669 max_rank=6 min=65317 min_rank=2 imbalance_ratio=1.002029 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[65250, 65656, 65209, 65947, 65435, 65921, 65573, 65297] avg=65536.0000 max=65947 max_rank=3 min=65209 min_rank=2 imbalance_ratio=1.006271 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[65455, 65883, 65300, 65315, 65376, 65769, 65562, 65628] avg=65536.0000 max=65883 max_rank=1 min=65300 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[65711, 65031, 65732, 65771, 65565, 65495, 65493, 65490] avg=65536.0000 max=65771 max_rank=3 min=65031 min_rank=1 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[65797, 65385, 65486, 65278, 65972, 65564, 65200, 65606] avg=65536.0000 max=65972 max_rank=4 min=65200 min_rank=6 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=2097386 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508288 peak_demand=66032 required_capacity=1.007568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=2098319 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506752 peak_demand=66149 required_capacity=1.009354 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=2096609 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508160 peak_demand=66006 required_capacity=1.007172 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=2096655 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508032 peak_demand=65831 required_capacity=1.004501 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=2098606 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506624 peak_demand=66036 required_capacity=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=2097451 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507520 peak_demand=66014 required_capacity=1.007294 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=2094877 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509184 peak_demand=65844 required_capacity=1.004700 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=2097313 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508288 peak_demand=66036 required_capacity=1.007629 + [2026-08-30 05:16:40] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 471.6 | throughput per GPU (TFLOP/s/GPU): 706.9 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.103357E+00 | loss scale: 1.0 | grad norm: 5.324 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66560 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=81920 expected_recv=131079 expected_overflow=0 exact_recv_per_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=81920 expected_recv=131095 expected_overflow=0 exact_recv_per_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=81920 expected_recv=131102 expected_overflow=0 exact_recv_per_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=81920 expected_recv=131310 expected_overflow=0 exact_recv_per_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=81920 expected_recv=130929 expected_overflow=0 exact_recv_per_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=81920 expected_recv=131105 expected_overflow=0 exact_recv_per_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=81920 expected_recv=130994 expected_overflow=0 exact_recv_per_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=81920 expected_recv=131079 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=81920 expected_recv=131095 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=81920 expected_recv=131102 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=81920 expected_recv=130962 expected_overflow=0 exact_recv_per_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=81920 expected_recv=131310 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=81920 expected_recv=130929 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=81920 expected_recv=131105 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=81920 expected_recv=130994 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=81920 expected_recv=130962 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=81920 expected_recv=130837 expected_overflow=0 exact_recv_per_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=81920 expected_recv=130837 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131295 expected_overflow=0 exact_recv_per_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=81920 expected_recv=130618 expected_overflow=0 exact_recv_per_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131274 expected_overflow=0 exact_recv_per_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131123 expected_overflow=0 exact_recv_per_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131056 expected_overflow=0 exact_recv_per_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131295 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=81920 expected_recv=130618 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131274 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131123 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131187 expected_overflow=0 exact_recv_per_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131056 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131186 expected_overflow=0 exact_recv_per_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131187 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=81920 expected_recv=131186 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131325 expected_overflow=0 exact_recv_per_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131325 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=81920 expected_recv=130378 expected_overflow=0 exact_recv_per_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131219 expected_overflow=0 exact_recv_per_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131107 expected_overflow=0 exact_recv_per_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=81920 expected_recv=130969 expected_overflow=0 exact_recv_per_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=81920 expected_recv=130378 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131219 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131716 expected_overflow=0 exact_recv_per_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131022 expected_overflow=0 exact_recv_per_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=81920 expected_recv=130840 expected_overflow=0 exact_recv_per_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131107 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=81920 expected_recv=130969 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131716 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=81920 expected_recv=131022 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=81920 expected_recv=130840 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=81920 expected_recv=130926 expected_overflow=0 exact_recv_per_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=81920 expected_recv=130926 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131085 expected_overflow=0 exact_recv_per_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131042 expected_overflow=0 exact_recv_per_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131421 expected_overflow=0 exact_recv_per_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131407 expected_overflow=0 exact_recv_per_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131085 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131216 expected_overflow=0 exact_recv_per_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=81920 expected_recv=130436 expected_overflow=0 exact_recv_per_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131042 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131043 expected_overflow=0 exact_recv_per_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131421 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131407 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131216 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=81920 expected_recv=130436 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=81920 expected_recv=131043 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=81920 expected_recv=130677 expected_overflow=0 exact_recv_per_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=81920 expected_recv=130677 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=81920 expected_recv=130566 expected_overflow=0 exact_recv_per_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131833 expected_overflow=0 exact_recv_per_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131024 expected_overflow=0 exact_recv_per_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=81920 expected_recv=130566 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131833 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=81920 expected_recv=130645 expected_overflow=0 exact_recv_per_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131024 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131550 expected_overflow=0 exact_recv_per_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131109 expected_overflow=0 exact_recv_per_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131172 expected_overflow=0 exact_recv_per_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=81920 expected_recv=130645 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131550 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131109 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=81920 expected_recv=131172 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=81920 expected_recv=130872 expected_overflow=0 exact_recv_per_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=81920 expected_recv=130872 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=81920 expected_recv=130963 expected_overflow=0 exact_recv_per_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=81920 expected_recv=131280 expected_overflow=0 exact_recv_per_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=81920 expected_recv=130963 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=81920 expected_recv=130594 expected_overflow=0 exact_recv_per_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=81920 expected_recv=131280 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=81920 expected_recv=130956 expected_overflow=0 exact_recv_per_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=81920 expected_recv=130594 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=81920 expected_recv=131430 expected_overflow=0 exact_recv_per_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=81920 expected_recv=131025 expected_overflow=0 exact_recv_per_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=81920 expected_recv=131456 expected_overflow=0 exact_recv_per_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=81920 expected_recv=130956 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=81920 expected_recv=131430 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=81920 expected_recv=131025 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130900 expected_overflow=0 exact_recv_per_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=81920 expected_recv=131456 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130900 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130913 expected_overflow=0 exact_recv_per_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130913 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130929 expected_overflow=0 exact_recv_per_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130921 expected_overflow=0 exact_recv_per_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=81920 expected_recv=131528 expected_overflow=0 exact_recv_per_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=81920 expected_recv=131069 expected_overflow=0 exact_recv_per_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130929 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130921 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=81920 expected_recv=131497 expected_overflow=0 exact_recv_per_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130819 expected_overflow=0 exact_recv_per_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=81920 expected_recv=131528 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=81920 expected_recv=131069 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131463 expected_overflow=0 exact_recv_per_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=81920 expected_recv=131497 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=81920 expected_recv=130819 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131463 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131372 expected_overflow=0 exact_recv_per_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131372 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131139 expected_overflow=0 exact_recv_per_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131378 expected_overflow=0 exact_recv_per_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=81920 expected_recv=130843 expected_overflow=0 exact_recv_per_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131118 expected_overflow=0 exact_recv_per_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131139 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131378 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=81920 expected_recv=130843 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131118 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131027 expected_overflow=0 exact_recv_per_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=81920 expected_recv=130236 expected_overflow=0 exact_recv_per_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131341 expected_overflow=0 exact_recv_per_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=81920 expected_recv=131027 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=81920 expected_recv=130236 actual_valid_recv=131200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131341 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=81920 expected_recv=130986 expected_overflow=0 exact_recv_per_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131200 slack_slots=32640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=81920 expected_recv=130986 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131120 expected_overflow=0 exact_recv_per_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131126 expected_overflow=0 exact_recv_per_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=81920 expected_recv=130965 expected_overflow=0 exact_recv_per_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131120 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131150 expected_overflow=0 exact_recv_per_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131126 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=81920 expected_recv=130965 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=81920 expected_recv=131150 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=81920 expected_recv=130893 expected_overflow=0 exact_recv_per_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=81920 expected_recv=130995 expected_overflow=0 exact_recv_per_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=81920 expected_recv=130978 expected_overflow=0 exact_recv_per_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=81920 expected_recv=130893 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=81920 expected_recv=130995 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131139 expected_overflow=0 exact_recv_per_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=81920 expected_recv=130978 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131139 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131133 expected_overflow=0 exact_recv_per_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131048 expected_overflow=0 exact_recv_per_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=81920 expected_recv=130921 expected_overflow=0 exact_recv_per_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131133 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131048 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131126 expected_overflow=0 exact_recv_per_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=81920 expected_recv=130921 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=81920 expected_recv=130963 expected_overflow=0 exact_recv_per_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131126 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131268 expected_overflow=0 exact_recv_per_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=81920 expected_recv=130914 expected_overflow=0 exact_recv_per_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=81920 expected_recv=130963 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=81920 expected_recv=131268 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=81920 expected_recv=130955 expected_overflow=0 exact_recv_per_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=81920 expected_recv=130914 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=81920 expected_recv=130955 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=81920 expected_recv=130957 expected_overflow=0 exact_recv_per_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=81920 expected_recv=131463 expected_overflow=0 exact_recv_per_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=81920 expected_recv=130890 expected_overflow=0 exact_recv_per_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=81920 expected_recv=130957 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=81920 expected_recv=131463 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=81920 expected_recv=131051 expected_overflow=0 exact_recv_per_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=81920 expected_recv=130890 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=81920 expected_recv=131377 expected_overflow=0 exact_recv_per_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=81920 expected_recv=131051 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=81920 expected_recv=130969 expected_overflow=0 exact_recv_per_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131075 expected_overflow=0 exact_recv_per_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=81920 expected_recv=131377 actual_valid_recv=132736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=81920 expected_recv=130969 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131075 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=81920 expected_recv=130996 expected_overflow=0 exact_recv_per_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132736 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=81920 expected_recv=130996 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131374 expected_overflow=0 exact_recv_per_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=81920 expected_recv=130857 expected_overflow=0 exact_recv_per_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131023 expected_overflow=0 exact_recv_per_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131374 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=81920 expected_recv=130857 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=81920 expected_recv=130878 expected_overflow=0 exact_recv_per_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131023 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=81920 expected_recv=130878 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131323 expected_overflow=0 exact_recv_per_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131050 expected_overflow=0 exact_recv_per_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131159 expected_overflow=0 exact_recv_per_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131323 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131382 expected_overflow=0 exact_recv_per_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=81920 expected_recv=131050 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131159 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131382 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131061 expected_overflow=0 exact_recv_per_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131256 expected_overflow=0 exact_recv_per_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131061 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=81920 expected_recv=130956 expected_overflow=0 exact_recv_per_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131256 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131117 expected_overflow=0 exact_recv_per_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=81920 expected_recv=130956 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=81920 expected_recv=131117 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=81920 expected_recv=130805 expected_overflow=0 exact_recv_per_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=81920 expected_recv=130547 expected_overflow=0 exact_recv_per_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=81920 expected_recv=130840 expected_overflow=0 exact_recv_per_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=81920 expected_recv=130805 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131309 expected_overflow=0 exact_recv_per_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=81920 expected_recv=130547 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=81920 expected_recv=130840 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131309 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=81920 expected_recv=130729 expected_overflow=0 exact_recv_per_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131103 expected_overflow=0 exact_recv_per_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=81920 expected_recv=130729 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131140 expected_overflow=0 exact_recv_per_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=81920 expected_recv=130696 expected_overflow=0 exact_recv_per_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131103 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131140 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=81920 expected_recv=130696 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131238 expected_overflow=0 exact_recv_per_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=81920 expected_recv=130829 expected_overflow=0 exact_recv_per_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131814 expected_overflow=0 exact_recv_per_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131238 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=81920 expected_recv=130930 expected_overflow=0 exact_recv_per_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=81920 expected_recv=130829 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=81920 expected_recv=131814 actual_valid_recv=132864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=81920 expected_recv=130930 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=81920 expected_recv=131531 expected_overflow=0 exact_recv_per_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132864 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=81920 expected_recv=130770 expected_overflow=0 exact_recv_per_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=81920 expected_recv=130925 expected_overflow=0 exact_recv_per_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=81920 expected_recv=131531 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=81920 expected_recv=131199 expected_overflow=0 exact_recv_per_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=81920 expected_recv=130770 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=81920 expected_recv=130925 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=81920 expected_recv=131199 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=81920 expected_recv=130800 expected_overflow=0 exact_recv_per_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131428 expected_overflow=0 exact_recv_per_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=81920 expected_recv=131592 expected_overflow=0 exact_recv_per_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=81920 expected_recv=130800 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131428 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131069 expected_overflow=0 exact_recv_per_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=81920 expected_recv=131592 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131487 expected_overflow=0 exact_recv_per_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131069 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131308 expected_overflow=0 exact_recv_per_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=81920 expected_recv=130721 expected_overflow=0 exact_recv_per_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131487 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131025 expected_overflow=0 exact_recv_per_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131308 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=81920 expected_recv=130721 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131025 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=81920 expected_recv=130264 expected_overflow=0 exact_recv_per_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131274 expected_overflow=0 exact_recv_per_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=81920 expected_recv=130264 actual_valid_recv=131328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=81920 expected_recv=131274 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131328 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[65405, 65375, 65551, 65497, 65552, 65817, 65721, 65370] avg=65536.0000 max=65817 max_rank=5 min=65370 min_rank=7 imbalance_ratio=1.004288 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[65901, 65333, 65436, 65670, 65486, 65538, 65422, 65502] avg=65536.0000 max=65901 max_rank=0 min=65333 min_rank=1 imbalance_ratio=1.005569 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[65623, 65866, 65058, 65881, 65696, 65291, 65531, 65342] avg=65536.0000 max=65881 max_rank=3 min=65058 min_rank=2 imbalance_ratio=1.005264 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[65503, 65425, 65777, 65734, 65318, 65459, 65398, 65674] avg=65536.0000 max=65777 max_rank=2 min=65318 min_rank=4 imbalance_ratio=1.003677 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[65839, 65228, 65310, 65519, 65729, 65531, 65377, 65755] avg=65536.0000 max=65839 max_rank=0 min=65228 min_rank=1 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65500, 65411, 65496, 65371, 65677, 65765, 65793, 65275] avg=65536.0000 max=65793 max_rank=6 min=65275 min_rank=7 imbalance_ratio=1.003922 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=93 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[65292, 65657, 65351, 65861, 65750, 65683, 65218, 65476] avg=65536.0000 max=65861 max_rank=3 min=65218 min_rank=6 imbalance_ratio=1.004959 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=93 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=93 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=93 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=93 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=93 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=93 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[65842, 65694, 65180, 65521, 65792, 65781, 65008, 65470] avg=65536.0000 max=65842 max_rank=0 min=65008 min_rank=6 imbalance_ratio=1.004669 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=93 active_clone_links=1 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[65539, 65607, 65317, 65511, 65552, 65942, 65119, 65701] avg=65536.0000 max=65942 max_rank=5 min=65119 min_rank=6 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[65462, 65578, 65506, 65610, 65644, 65399, 65749, 65340] avg=65536.0000 max=65749 max_rank=6 min=65340 min_rank=7 imbalance_ratio=1.003250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[65400, 65371, 65354, 65686, 65537, 65746, 65448, 65746] avg=65536.0000 max=65746 max_rank=5 min=65354 min_rank=2 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[65846, 65470, 65493, 65725, 65550, 65454, 65282, 65468] avg=65536.0000 max=65846 max_rank=0 min=65282 min_rank=6 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[65498, 65711, 65761, 65489, 65616, 65397, 65341, 65475] avg=65536.0000 max=65761 max_rank=2 min=65341 min_rank=6 imbalance_ratio=1.003433 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[65281, 65373, 65590, 65703, 65306, 65608, 65828, 65599] avg=65536.0000 max=65828 max_rank=6 min=65281 min_rank=0 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[65801, 65357, 65533, 65488, 65713, 65483, 65759, 65154] avg=65536.0000 max=65801 max_rank=0 min=65154 min_rank=7 imbalance_ratio=1.004044 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[65824, 65469, 65280, 65181, 65562, 65412, 65849, 65711] avg=65536.0000 max=65849 max_rank=6 min=65181 min_rank=3 imbalance_ratio=1.004776 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=2099208 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=505600 peak_demand=65994 required_capacity=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=2096373 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508032 peak_demand=65959 required_capacity=1.006454 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=2097604 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508288 peak_demand=65908 required_capacity=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=2096695 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507520 peak_demand=65948 required_capacity=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=2096611 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508160 peak_demand=65821 required_capacity=1.004349 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=2095756 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509568 peak_demand=66192 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=2097731 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508160 peak_demand=65986 required_capacity=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=2097238 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507264 peak_demand=65881 required_capacity=1.005264 + [2026-08-30 05:16:41] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 472.0 | throughput per GPU (TFLOP/s/GPU): 706.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.984506E+00 | loss scale: 1.0 | grad norm: 5.125 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66560 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131019 expected_overflow=0 exact_recv_per_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=81920 expected_recv=130753 expected_overflow=0 exact_recv_per_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131108 expected_overflow=0 exact_recv_per_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=81920 expected_recv=130788 expected_overflow=0 exact_recv_per_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131198 expected_overflow=0 exact_recv_per_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131145 expected_overflow=0 exact_recv_per_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131309 expected_overflow=0 exact_recv_per_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131019 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=81920 expected_recv=130753 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131108 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=81920 expected_recv=130788 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131198 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131256 expected_overflow=0 exact_recv_per_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131145 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131309 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=81920 expected_recv=131256 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=81920 expected_recv=130823 expected_overflow=0 exact_recv_per_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131110 expected_overflow=0 exact_recv_per_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131261 expected_overflow=0 exact_recv_per_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=81920 expected_recv=130427 expected_overflow=0 exact_recv_per_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=81920 expected_recv=130823 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131160 expected_overflow=0 exact_recv_per_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131110 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131261 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=81920 expected_recv=130427 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131356 expected_overflow=0 exact_recv_per_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131160 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131160 expected_overflow=0 exact_recv_per_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131356 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131160 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131279 expected_overflow=0 exact_recv_per_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=81920 expected_recv=130775 expected_overflow=0 exact_recv_per_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=81920 expected_recv=131279 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131533 expected_overflow=0 exact_recv_per_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131328 expected_overflow=0 exact_recv_per_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=81920 expected_recv=130815 expected_overflow=0 exact_recv_per_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=81920 expected_recv=130775 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131533 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131328 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=81920 expected_recv=130815 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131093 expected_overflow=0 exact_recv_per_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131569 expected_overflow=0 exact_recv_per_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131007 expected_overflow=0 exact_recv_per_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=81920 expected_recv=130456 expected_overflow=0 exact_recv_per_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131093 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131081 expected_overflow=0 exact_recv_per_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131569 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=81920 expected_recv=131007 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=81920 expected_recv=130921 expected_overflow=0 exact_recv_per_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=81920 expected_recv=130456 actual_valid_recv=131456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131028 expected_overflow=0 exact_recv_per_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131081 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=81920 expected_recv=130921 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=163840 valid_num_tokens=131456 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131028 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131053 expected_overflow=0 exact_recv_per_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131053 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131198 expected_overflow=0 exact_recv_per_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131176 expected_overflow=0 exact_recv_per_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131288 expected_overflow=0 exact_recv_per_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131198 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=81920 expected_recv=130831 expected_overflow=0 exact_recv_per_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131176 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=81920 expected_recv=131288 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=81920 expected_recv=130831 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=81920 expected_recv=131373 expected_overflow=0 exact_recv_per_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=81920 expected_recv=131260 expected_overflow=0 exact_recv_per_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=81920 expected_recv=131175 expected_overflow=0 exact_recv_per_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=81920 expected_recv=131373 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=81920 expected_recv=131260 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130968 expected_overflow=0 exact_recv_per_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=81920 expected_recv=131175 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130968 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130760 expected_overflow=0 exact_recv_per_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130951 expected_overflow=0 exact_recv_per_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130760 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130951 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130906 expected_overflow=0 exact_recv_per_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=81920 expected_recv=131183 expected_overflow=0 exact_recv_per_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=81920 expected_recv=130906 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131026 expected_overflow=0 exact_recv_per_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=81920 expected_recv=131183 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131219 expected_overflow=0 exact_recv_per_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=81920 expected_recv=130891 expected_overflow=0 exact_recv_per_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131003 expected_overflow=0 exact_recv_per_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131026 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131219 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=81920 expected_recv=130891 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131003 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=81920 expected_recv=130870 expected_overflow=0 exact_recv_per_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131334 expected_overflow=0 exact_recv_per_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=81920 expected_recv=130870 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131334 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131251 expected_overflow=0 exact_recv_per_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=81920 expected_recv=130982 expected_overflow=0 exact_recv_per_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=81920 expected_recv=131226 expected_overflow=0 exact_recv_per_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=81920 expected_recv=131251 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=81920 expected_recv=131159 expected_overflow=0 exact_recv_per_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=81920 expected_recv=130982 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=81920 expected_recv=131226 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=81920 expected_recv=130665 expected_overflow=0 exact_recv_per_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=81920 expected_recv=131537 expected_overflow=0 exact_recv_per_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=81920 expected_recv=131159 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=81920 expected_recv=130665 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=81920 expected_recv=131537 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=81920 expected_recv=130823 expected_overflow=0 exact_recv_per_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=81920 expected_recv=131410 expected_overflow=0 exact_recv_per_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=81920 expected_recv=130823 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=81920 expected_recv=131410 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=81920 expected_recv=130827 expected_overflow=0 exact_recv_per_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=81920 expected_recv=130929 expected_overflow=0 exact_recv_per_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=81920 expected_recv=131417 expected_overflow=0 exact_recv_per_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=81920 expected_recv=130750 expected_overflow=0 exact_recv_per_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=81920 expected_recv=130827 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=81920 expected_recv=130929 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=81920 expected_recv=131417 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=81920 expected_recv=130750 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=81920 expected_recv=130908 expected_overflow=0 exact_recv_per_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=81920 expected_recv=131404 expected_overflow=0 exact_recv_per_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=81920 expected_recv=130908 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=81920 expected_recv=131404 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=81920 expected_recv=130988 expected_overflow=0 exact_recv_per_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=81920 expected_recv=130562 expected_overflow=0 exact_recv_per_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=81920 expected_recv=130988 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=81920 expected_recv=130562 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=81920 expected_recv=131316 expected_overflow=0 exact_recv_per_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=81920 expected_recv=131231 expected_overflow=0 exact_recv_per_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=81920 expected_recv=130695 expected_overflow=0 exact_recv_per_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=81920 expected_recv=131316 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131322 expected_overflow=0 exact_recv_per_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=81920 expected_recv=131231 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=81920 expected_recv=130695 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131322 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=81920 expected_recv=130990 expected_overflow=0 exact_recv_per_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=81920 expected_recv=130968 expected_overflow=0 exact_recv_per_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131034 expected_overflow=0 exact_recv_per_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=81920 expected_recv=130990 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=81920 expected_recv=130968 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131034 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131580 expected_overflow=0 exact_recv_per_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=81920 expected_recv=130835 expected_overflow=0 exact_recv_per_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131580 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131152 expected_overflow=0 exact_recv_per_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=81920 expected_recv=130835 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131187 expected_overflow=0 exact_recv_per_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131514 expected_overflow=0 exact_recv_per_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=81920 expected_recv=131152 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131187 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131514 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131107 expected_overflow=0 exact_recv_per_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131186 expected_overflow=0 exact_recv_per_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=81920 expected_recv=130972 expected_overflow=0 exact_recv_per_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131107 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131186 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=81920 expected_recv=130964 expected_overflow=0 exact_recv_per_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=81920 expected_recv=130972 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=81920 expected_recv=130964 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=81920 expected_recv=130622 expected_overflow=0 exact_recv_per_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131024 expected_overflow=0 exact_recv_per_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=81920 expected_recv=131312 expected_overflow=0 exact_recv_per_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=81920 expected_recv=130622 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=81920 expected_recv=131219 expected_overflow=0 exact_recv_per_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=81920 expected_recv=131312 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=81920 expected_recv=131024 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=81920 expected_recv=131219 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=81920 expected_recv=130880 expected_overflow=0 exact_recv_per_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=81920 expected_recv=131266 expected_overflow=0 exact_recv_per_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=81920 expected_recv=130880 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=81920 expected_recv=131370 expected_overflow=0 exact_recv_per_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=81920 expected_recv=131266 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=81920 expected_recv=130884 expected_overflow=0 exact_recv_per_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=81920 expected_recv=131370 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=81920 expected_recv=130884 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=81920 expected_recv=130838 expected_overflow=0 exact_recv_per_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=81920 expected_recv=130860 expected_overflow=0 exact_recv_per_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=81920 expected_recv=130807 expected_overflow=0 exact_recv_per_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131132 expected_overflow=0 exact_recv_per_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=81920 expected_recv=130838 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=81920 expected_recv=130860 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=81920 expected_recv=130807 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131132 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131131 expected_overflow=0 exact_recv_per_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131004 expected_overflow=0 exact_recv_per_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=81920 expected_recv=130845 expected_overflow=0 exact_recv_per_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131131 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131004 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131372 expected_overflow=0 exact_recv_per_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=81920 expected_recv=130845 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131372 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131260 expected_overflow=0 exact_recv_per_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=81920 expected_recv=130969 expected_overflow=0 exact_recv_per_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=81920 expected_recv=130972 expected_overflow=0 exact_recv_per_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=81920 expected_recv=131260 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=81920 expected_recv=130969 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=81920 expected_recv=130660 expected_overflow=0 exact_recv_per_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=81920 expected_recv=130972 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=81920 expected_recv=130660 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=81920 expected_recv=130902 expected_overflow=0 exact_recv_per_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=81920 expected_recv=131128 expected_overflow=0 exact_recv_per_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=81920 expected_recv=131353 expected_overflow=0 exact_recv_per_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=81920 expected_recv=131082 expected_overflow=0 exact_recv_per_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=81920 expected_recv=130902 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=81920 expected_recv=131128 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=81920 expected_recv=131353 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=81920 expected_recv=131082 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=81920 expected_recv=131517 expected_overflow=0 exact_recv_per_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131041 expected_overflow=0 exact_recv_per_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=81920 expected_recv=130965 expected_overflow=0 exact_recv_per_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=81920 expected_recv=131517 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131041 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=81920 expected_recv=130965 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=81920 expected_recv=130878 expected_overflow=0 exact_recv_per_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131410 expected_overflow=0 exact_recv_per_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=81920 expected_recv=130824 expected_overflow=0 exact_recv_per_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=81920 expected_recv=130675 expected_overflow=0 exact_recv_per_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=81920 expected_recv=130878 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131410 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=81920 expected_recv=130824 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=81920 expected_recv=130675 actual_valid_recv=131584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=81920 expected_recv=130799 expected_overflow=0 exact_recv_per_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131184 expected_overflow=0 exact_recv_per_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131584 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131093 expected_overflow=0 exact_recv_per_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=81920 expected_recv=130799 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131184 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131093 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131471 expected_overflow=0 exact_recv_per_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131467 expected_overflow=0 exact_recv_per_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131327 expected_overflow=0 exact_recv_per_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131164 expected_overflow=0 exact_recv_per_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131174 expected_overflow=0 exact_recv_per_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131467 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=81920 expected_recv=131471 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131327 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131164 actual_valid_recv=132096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131174 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131298 expected_overflow=0 exact_recv_per_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=81920 expected_recv=130896 expected_overflow=0 exact_recv_per_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131081 expected_overflow=0 exact_recv_per_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132096 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131298 actual_valid_recv=132352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=81920 expected_recv=130896 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=81920 expected_recv=131081 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132352 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=81920 expected_recv=130887 expected_overflow=0 exact_recv_per_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131283 expected_overflow=0 exact_recv_per_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131615 expected_overflow=0 exact_recv_per_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=81920 expected_recv=130887 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=81920 expected_recv=130840 expected_overflow=0 exact_recv_per_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131283 actual_valid_recv=132608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131615 actual_valid_recv=132480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131040 expected_overflow=0 exact_recv_per_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=81920 expected_recv=130807 expected_overflow=0 exact_recv_per_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=81920 expected_recv=130840 actual_valid_recv=131712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132480 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132608 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=81920 expected_recv=131040 actual_valid_recv=132224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=81920 expected_recv=130807 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131712 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=81920 expected_recv=130668 expected_overflow=0 exact_recv_per_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=132224 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=81920 expected_recv=130668 actual_valid_recv=131968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=163840 valid_num_tokens=131968 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=81920 expected_recv=130806 expected_overflow=0 exact_recv_per_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=81920 expected_recv=130806 actual_valid_recv=131840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=163840 valid_num_tokens=131840 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[65206, 65456, 64989, 65595, 65598, 65598, 65774, 66072] avg=65536.0000 max=66072 max_rank=7 min=64989 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[65238, 65188, 65564, 65652, 65859, 65762, 65360, 65665] avg=65536.0000 max=65859 max_rank=4 min=65188 min_rank=1 imbalance_ratio=1.004929 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[65455, 65660, 65651, 65285, 65270, 65740, 65519, 65708] avg=65536.0000 max=65740 max_rank=5 min=65270 min_rank=4 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[65374, 65451, 65726, 65689, 65202, 65567, 65773, 65506] avg=65536.0000 max=65773 max_rank=6 min=65202 min_rank=4 imbalance_ratio=1.003616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[65714, 65627, 65654, 65370, 65581, 65697, 65179, 65466] avg=65536.0000 max=65714 max_rank=0 min=65179 min_rank=6 imbalance_ratio=1.002716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65425, 65553, 65352, 65354, 65541, 65622, 65487, 65954] avg=65536.0000 max=65954 max_rank=7 min=65352 min_rank=2 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[65520, 65576, 65288, 65474, 65353, 65525, 65494, 66058] avg=65536.0000 max=66058 max_rank=7 min=65288 min_rank=2 imbalance_ratio=1.007965 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[65877, 65435, 65404, 65624, 65657, 65328, 65476, 65487] avg=65536.0000 max=65877 max_rank=0 min=65328 min_rank=5 imbalance_ratio=1.005203 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[65717, 65528, 65298, 65510, 65823, 65420, 65305, 65687] avg=65536.0000 max=65823 max_rank=4 min=65298 min_rank=2 imbalance_ratio=1.004379 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[65682, 65691, 65538, 65391, 65564, 65746, 65229, 65447] avg=65536.0000 max=65746 max_rank=5 min=65229 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[65459, 65485, 65316, 65914, 65222, 65694, 65726, 65472] avg=65536.0000 max=65914 max_rank=3 min=65222 min_rank=4 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[65822, 65546, 65313, 65486, 65423, 65406, 65645, 65647] avg=65536.0000 max=65822 max_rank=0 min=65313 min_rank=2 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[65663, 65456, 65153, 65604, 65281, 65410, 65969, 65752] avg=65536.0000 max=65969 max_rank=6 min=65153 min_rank=2 imbalance_ratio=1.006607 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[65832, 65309, 65493, 65442, 65899, 65183, 65506, 65624] avg=65536.0000 max=65899 max_rank=4 min=65183 min_rank=5 imbalance_ratio=1.005539 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[65357, 65631, 65926, 65562, 65625, 65267, 65286, 65634] avg=65536.0000 max=65926 max_rank=2 min=65267 min_rank=5 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[65128, 65833, 65927, 65156, 65129, 65630, 65567, 65918] avg=65536.0000 max=65927 max_rank=2 min=65128 min_rank=0 imbalance_ratio=1.005966 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=2097274 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509056 peak_demand=65858 required_capacity=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=2098849 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507136 peak_demand=66072 required_capacity=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=2097003 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508160 peak_demand=66101 required_capacity=1.008621 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=2096762 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509184 peak_demand=65975 required_capacity=1.006699 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=2097264 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=507904 peak_demand=66004 required_capacity=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=2096492 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=509184 peak_demand=65914 required_capacity=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=2096040 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=508800 peak_demand=65899 required_capacity=1.005539 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=2097532 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=506880 peak_demand=65877 required_capacity=1.005203 + [2026-08-30 05:16:41] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 470.6 | throughput per GPU (TFLOP/s/GPU): 708.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.866780E+00 | loss scale: 1.0 | grad norm: 5.169 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 66560 | min safe capacity: 1.0156 | +[after training is done] datetime: 2026-08-30 05:16:41 +[rank0]:[W830 05:16:42.173510945 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 05:16:44.581962874 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 05:16:44.647388254 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 05:16:44.648078044 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 05:16:44.849588293 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 05:16:44.850227808 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 05:16:44.911648381 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 05:16:44.175613436 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:16:49.953000 502477 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:16:49.954000 502477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 502572 closing signal SIGTERM +W0830 05:16:49.954000 502477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 502574 closing signal SIGTERM +W0830 05:16:49.954000 502477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 502575 closing signal SIGTERM +W0830 05:16:49.955000 502477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 502576 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 502477 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..42c2a14ede900c7bc64225dd7703aeea863e4b6e --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 47 +2, 629 +3, 629 +4, 629 +5, 629 +6, 47 +7, 629 +0, 893 +1, 889 +2, 901 +3, 897 +4, 899 +5, 903 +6, 931 +7, 925 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 2387 +1, 2385 +2, 2387 +3, 2381 +4, 2409 +5, 2397 +6, 2397 +7, 2409 +0, 12469 +1, 12249 +2, 12701 +3, 12933 +4, 12389 +5, 12013 +6, 12553 +7, 12641 +0, 42989 +1, 43437 +2, 43437 +3, 43437 +4, 42989 +5, 43437 +6, 43437 +7, 43437 +0, 43741 +1, 44189 +2, 44189 +3, 44187 +4, 43741 +5, 44187 +6, 44189 +7, 44189 +0, 44429 +1, 44877 +2, 44877 +3, 44877 +4, 44429 +5, 44877 +6, 44877 +7, 44877 +0, 45551 +1, 45999 +2, 45999 +3, 45999 +4, 45551 +5, 45999 +6, 45999 +7, 45999 +0, 47173 +1, 47621 +2, 47621 +3, 47621 +4, 47173 +5, 47621 +6, 47621 +7, 48751 +0, 48303 +1, 48751 +2, 48751 +3, 48751 +4, 48303 +5, 48751 +6, 48751 +7, 48751 +0, 79811 +1, 79807 +2, 79807 +3, 79807 +4, 79831 +5, 79807 +6, 79807 +7, 79807 +0, 79811 +1, 79807 +2, 79807 +3, 79807 +4, 79831 +5, 79807 +6, 79807 +7, 79807 +0, 80935 +1, 80931 +2, 80931 +3, 80931 +4, 80955 +5, 80931 +6, 80931 +7, 80931 +0, 82247 +1, 82257 +2, 82247 +3, 82249 +4, 82287 +5, 82257 +6, 82277 +7, 82263 +0, 92769 +1, 92825 +2, 92813 +3, 92833 +4, 92889 +5, 92823 +6, 92801 +7, 92851 +0, 93829 +1, 93821 +2, 93837 +3, 93895 +4, 93889 +5, 93907 +6, 93865 +7, 93911 +0, 109059 +1, 109393 +2, 109113 +3, 109233 +4, 109217 +5, 109193 +6, 109313 +7, 109313 +0, 117507 +1, 117581 +2, 117581 +3, 117581 +4, 118667 +5, 117581 +6, 117581 +7, 117581 +0, 117507 +1, 117581 +2, 117581 +3, 117581 +4, 118667 +5, 117581 +6, 117581 +7, 117581 +0, 117507 +1, 117581 +2, 117581 +3, 117581 +4, 118667 +5, 117581 +6, 117581 +7, 117581 +0, 113993 +1, 114067 +2, 114067 +3, 114067 +4, 115153 +5, 114067 +6, 114067 +7, 114067 +0, 112313 +1, 112387 +2, 112387 +3, 112387 +4, 113473 +5, 112387 +6, 112387 +7, 112387 +0, 1 +1, 4 +2, 4 +3, 4 +4, 132 +5, 111999 +6, 644 +7, 4 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..9922c76ef3083645e5102beb45490cca12af1bab --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G4 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=896 +num_experts=128 +topk=32 +experts_per_rank=16 +expert_params=11010048 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=131072 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.095703125 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0b192896deb9ef07e2be9a017581eea66a3a016a --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/train.log @@ -0,0 +1,9351 @@ +[05:18:08] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G4 / ultraep (slots/rank=16 policy=matched) + E=128 K=32 H'=896 m=16 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.095703125 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:18:10.269000 509099 torch/distributed/run.py:766] +W0830 05:18:10.269000 509099 torch/distributed/run.py:766] ***************************************** +W0830 05:18:10.269000 509099 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:18:10.269000 509099 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 896 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.095703125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 32 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 67 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank6]:[W830 05:18:28.602786789 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:18:28.602854330 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:18:28.604213397 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:18:28.604269795 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:18:28.604271864 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:18:28.604310298 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:18:28.604331072 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:18:28.765297317 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.807 seconds +time to initialize megatron (seconds): 7.102 +[after megatron is initialized] datetime: 2026-08-30 05:18:34 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6728847360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42479616 elements, 42479616 padded size): + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight +Params for bucket 3 (42475520 elements, 42475520 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_proj.weight +Params for bucket 4 (42475520 elements, 42475520 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 5 (42475520 elements, 42475520 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_qkv.weight +Params for bucket 6 (42475520 elements, 42475520 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.pre_mlp_layernorm.weight +Params for bucket 7 (42475520 elements, 42475520 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight +Params for bucket 8 (42475520 elements, 42475520 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 9 (42475520 elements, 42475520 padded size): + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (42475520 elements, 42475520 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 11 (42475520 elements, 42475520 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight +Params for bucket 12 (42475520 elements, 42475520 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight +Params for bucket 13 (42475520 elements, 42475520 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.pre_mlp_layernorm.weight +Params for bucket 14 (42475520 elements, 42475520 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.weight +Params for bucket 15 (42475520 elements, 42475520 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight +Params for bucket 16 (42475520 elements, 42475520 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 17 (42475520 elements, 42475520 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 42 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 +Params for bucket 43 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 46 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 +Params for bucket 47 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 51 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 52 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 55 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 +Params for bucket 56 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 59 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 +Params for bucket 60 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 +Params for bucket 63 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 +Params for bucket 64 (40370176 elements, 40370176 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 +Params for bucket 66 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 67 (29360128 elements, 29360128 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:18:38 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:18:38 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3868.79, 3882.81) + train/valid/test-data-iterators-setup ..........: (71.83, 90.10) +[before the start of training step] datetime: 2026-08-30 05:18:38 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=143616 expected_recv=256154 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=143616 expected_recv=260948 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=143616 expected_recv=263153 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=143616 expected_recv=257833 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=143616 expected_recv=256154 actual_valid_recv=258176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=143616 expected_recv=260948 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=258176 slack_slots=29056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=143616 expected_recv=263153 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=143616 expected_recv=257833 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=265216 slack_slots=22016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=260096 slack_slots=27136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=143616 expected_recv=259978 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=143616 expected_recv=259978 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=143616 expected_recv=256210 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=143616 expected_recv=264374 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=143616 expected_recv=263713 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=143616 expected_recv=256210 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=143616 expected_recv=264374 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=258304 slack_slots=28928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=143616 expected_recv=263713 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=143616 expected_recv=259444 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=143616 expected_recv=259444 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=143616 expected_recv=255383 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=143616 expected_recv=255383 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=257920 slack_slots=29312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=143616 expected_recv=264381 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=143616 expected_recv=264381 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=143616 expected_recv=262556 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=143616 expected_recv=256332 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=143616 expected_recv=262556 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=143616 expected_recv=256332 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=264832 slack_slots=22400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=143616 expected_recv=256651 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=258688 slack_slots=28544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=143616 expected_recv=256651 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=143616 expected_recv=257181 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=259072 slack_slots=28160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=143616 expected_recv=257181 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=259328 slack_slots=27904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=143616 expected_recv=259490 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=143616 expected_recv=259490 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=143616 expected_recv=258578 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=143616 expected_recv=258404 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=143616 expected_recv=258578 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=143616 expected_recv=258404 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=260992 slack_slots=26240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=143616 expected_recv=259378 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=143616 expected_recv=255539 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=143616 expected_recv=259378 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=143616 expected_recv=255539 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=257408 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=143616 expected_recv=258494 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=143616 expected_recv=264526 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=143616 expected_recv=258494 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=143616 expected_recv=261582 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=260608 slack_slots=26624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=143616 expected_recv=264526 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=143616 expected_recv=261582 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=143616 expected_recv=257276 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=143616 expected_recv=257276 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=259456 slack_slots=27776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=143616 expected_recv=254815 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=143616 expected_recv=263865 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=143616 expected_recv=262064 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=143616 expected_recv=254815 actual_valid_recv=257024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=143616 expected_recv=263865 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=143616 expected_recv=260819 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=143616 expected_recv=260705 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=143616 expected_recv=262064 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=257024 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=143616 expected_recv=259806 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=265600 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=143616 expected_recv=260819 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=143616 expected_recv=256300 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=143616 expected_recv=260705 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=143616 expected_recv=259806 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=262912 slack_slots=24320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=143616 expected_recv=256300 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=259328 slack_slots=27904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=143616 expected_recv=261075 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=143616 expected_recv=261075 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=143616 expected_recv=258716 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=143616 expected_recv=258919 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263424 slack_slots=23808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=143616 expected_recv=258716 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=143616 expected_recv=258919 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=143616 expected_recv=257721 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=143616 expected_recv=259448 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=143616 expected_recv=259852 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=143616 expected_recv=257721 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=143616 expected_recv=259448 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=143616 expected_recv=261345 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=143616 expected_recv=264700 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=259840 slack_slots=27392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=143616 expected_recv=259852 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=143616 expected_recv=261345 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=143616 expected_recv=264700 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=143616 expected_recv=258766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=143616 expected_recv=258766 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=143616 expected_recv=259411 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=143616 expected_recv=262261 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=260864 slack_slots=26368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=143616 expected_recv=259411 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=143616 expected_recv=262261 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=143616 expected_recv=260120 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=143616 expected_recv=260120 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=143616 expected_recv=259231 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=262912 slack_slots=24320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=143616 expected_recv=259231 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=143616 expected_recv=264291 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=143616 expected_recv=264291 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=143616 expected_recv=259962 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=143616 expected_recv=260545 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=143616 expected_recv=259962 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=143616 expected_recv=255859 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=143616 expected_recv=263260 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=143616 expected_recv=260545 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=143616 expected_recv=255917 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=143616 expected_recv=255859 actual_valid_recv=258048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=143616 expected_recv=263260 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=143616 expected_recv=255917 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=258048 slack_slots=29184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=265600 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=143616 expected_recv=259642 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=258688 slack_slots=28544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=143616 expected_recv=259642 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=143616 expected_recv=261275 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=143616 expected_recv=261275 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=143616 expected_recv=259199 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=143616 expected_recv=254323 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=143616 expected_recv=259199 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=143616 expected_recv=261699 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=143616 expected_recv=254323 actual_valid_recv=256384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=143616 expected_recv=261699 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=256384 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=143616 expected_recv=257185 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=143616 expected_recv=262797 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=143616 expected_recv=257185 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=143616 expected_recv=262797 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=259072 slack_slots=28160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=265472 slack_slots=21760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=143616 expected_recv=258028 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=143616 expected_recv=257916 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=143616 expected_recv=258028 actual_valid_recv=260224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=143616 expected_recv=257916 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=260224 slack_slots=27008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=259968 slack_slots=27264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=143616 expected_recv=257342 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=143616 expected_recv=257732 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=143616 expected_recv=257342 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=143616 expected_recv=257732 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=259328 slack_slots=27904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=143616 expected_recv=261867 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=259840 slack_slots=27392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=143616 expected_recv=261867 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=143616 expected_recv=263712 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=143616 expected_recv=259847 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=143616 expected_recv=259345 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=143616 expected_recv=263712 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=143616 expected_recv=259847 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=143616 expected_recv=259345 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=143616 expected_recv=259695 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=143616 expected_recv=259695 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=4197711 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=361600 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=4196424 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=361728 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=4191694 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368256 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=4199140 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=359936 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=4194937 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=365696 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=4195375 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=363776 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=4196979 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364032 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=4182172 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377984 peak_demand=132384 required_capacity=1.010010 + [2026-08-30 05:19:07] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 28692.6 | throughput per GPU (TFLOP/s/GPU): 11.6 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102500E+01 | loss scale: 1.0 | grad norm: 44.122 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 133888 | min safe capacity: 1.0215 |Number of parameters in transformer block in billions: 23.22 + +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58076.1103515625 | max allocated: 83186.3115234375 | reserved: 85006.0 | max reserved: 85006.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=143616 expected_recv=262934 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=143616 expected_recv=255075 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=143616 expected_recv=257768 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=143616 expected_recv=262310 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=143616 expected_recv=262934 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=143616 expected_recv=255075 actual_valid_recv=257024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=143616 expected_recv=257768 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=265344 slack_slots=21888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=257024 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=143616 expected_recv=262310 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=260352 slack_slots=26880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=143616 expected_recv=264197 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=143616 expected_recv=261338 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=143616 expected_recv=264197 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=143616 expected_recv=261338 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=143616 expected_recv=260799 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=143616 expected_recv=263015 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=143616 expected_recv=260799 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=143616 expected_recv=263015 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=143616 expected_recv=253504 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=265216 slack_slots=22016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=143616 expected_recv=253504 actual_valid_recv=255360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=143616 expected_recv=259677 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=255360 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=143616 expected_recv=259677 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=143616 expected_recv=257763 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=143616 expected_recv=255884 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=143616 expected_recv=257763 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=259840 slack_slots=27392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=143616 expected_recv=259174 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=143616 expected_recv=255884 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=258304 slack_slots=28928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=143616 expected_recv=259174 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=143616 expected_recv=255090 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=143616 expected_recv=261939 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=143616 expected_recv=255090 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=143616 expected_recv=261939 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=257408 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=143616 expected_recv=261884 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=143616 expected_recv=261884 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=143616 expected_recv=256207 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=143616 expected_recv=256207 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=143616 expected_recv=263554 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=143616 expected_recv=258873 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=258688 slack_slots=28544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=143616 expected_recv=263554 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=143616 expected_recv=258873 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=143616 expected_recv=259455 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=143616 expected_recv=259455 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=143616 expected_recv=258628 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=143616 expected_recv=258628 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=143616 expected_recv=253734 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=260480 slack_slots=26752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=143616 expected_recv=253734 actual_valid_recv=255872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=143616 expected_recv=264084 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=255872 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=143616 expected_recv=264084 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=143616 expected_recv=259476 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=143616 expected_recv=261642 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=143616 expected_recv=259476 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=143616 expected_recv=261642 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=143616 expected_recv=261556 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=143616 expected_recv=261556 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=263424 slack_slots=23808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=143616 expected_recv=260036 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=143616 expected_recv=261716 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=143616 expected_recv=259012 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=143616 expected_recv=259913 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=143616 expected_recv=261716 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=143616 expected_recv=260036 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=143616 expected_recv=259012 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=143616 expected_recv=257022 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=143616 expected_recv=259913 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=143616 expected_recv=262795 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=143616 expected_recv=257022 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=143616 expected_recv=262795 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=143616 expected_recv=259418 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=265344 slack_slots=21888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=143616 expected_recv=259418 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=143616 expected_recv=269083 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=143616 expected_recv=269082 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=143616 expected_recv=269080 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=143616 expected_recv=254159 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=143616 expected_recv=269083 actual_valid_recv=271104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=143616 expected_recv=268616 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=143616 expected_recv=269082 actual_valid_recv=271488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=143616 expected_recv=269080 actual_valid_recv=271744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=271104 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=143616 expected_recv=254159 actual_valid_recv=256000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=143616 expected_recv=245820 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=143616 expected_recv=268616 actual_valid_recv=270720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=271488 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=271744 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=256000 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=143616 expected_recv=245820 actual_valid_recv=248320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=270720 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=248320 slack_slots=38912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=143616 expected_recv=261218 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=143616 expected_recv=252228 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=143616 expected_recv=261218 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=143616 expected_recv=262632 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=143616 expected_recv=252228 actual_valid_recv=254720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=143616 expected_recv=262632 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=143616 expected_recv=269084 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=254720 slack_slots=32512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=143616 expected_recv=269084 actual_valid_recv=270976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=143616 expected_recv=259975 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=270976 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=143616 expected_recv=259975 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=143616 expected_recv=259093 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=143616 expected_recv=259093 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=143616 expected_recv=258786 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=143616 expected_recv=258786 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=143616 expected_recv=260243 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=260992 slack_slots=26240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=143616 expected_recv=255526 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=143616 expected_recv=260243 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=143616 expected_recv=255526 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262912 slack_slots=24320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=143616 expected_recv=260510 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=257792 slack_slots=29440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=143616 expected_recv=260510 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=143616 expected_recv=258702 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=262912 slack_slots=24320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=143616 expected_recv=258702 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=143616 expected_recv=261932 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261120 slack_slots=26112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=143616 expected_recv=252641 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=143616 expected_recv=261932 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=143616 expected_recv=252641 actual_valid_recv=254848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=254848 slack_slots=32384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=143616 expected_recv=259513 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=143616 expected_recv=263006 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=143616 expected_recv=259513 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=143616 expected_recv=263006 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=143616 expected_recv=262589 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=143616 expected_recv=262589 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=143616 expected_recv=255856 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=265344 slack_slots=21888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=143616 expected_recv=255856 actual_valid_recv=258176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=143616 expected_recv=261230 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=258176 slack_slots=29056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=143616 expected_recv=261230 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=143616 expected_recv=254755 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=143616 expected_recv=262674 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=143616 expected_recv=254755 actual_valid_recv=256896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=143616 expected_recv=262674 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=256896 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=143616 expected_recv=264593 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=143616 expected_recv=259811 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=143616 expected_recv=264593 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=143616 expected_recv=259811 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=143616 expected_recv=259790 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=143616 expected_recv=254964 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=143616 expected_recv=259790 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=143616 expected_recv=254964 actual_valid_recv=256640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=143616 expected_recv=261659 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=256640 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=143616 expected_recv=262960 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=143616 expected_recv=261659 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=143616 expected_recv=262960 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=143616 expected_recv=261830 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=143616 expected_recv=264352 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=265472 slack_slots=21760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=143616 expected_recv=261830 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=143616 expected_recv=264352 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=143616 expected_recv=258632 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=143616 expected_recv=258632 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=143616 expected_recv=260001 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=143616 expected_recv=259920 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=143616 expected_recv=260001 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=143616 expected_recv=259920 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=143616 expected_recv=258943 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=143616 expected_recv=262574 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=143616 expected_recv=258943 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=143616 expected_recv=262574 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=4191399 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=367104 peak_demand=136700 required_capacity=1.042938 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=4199877 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=358912 peak_demand=136698 required_capacity=1.042923 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=4213936 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346496 peak_demand=136699 required_capacity=1.042931 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=4190415 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=369536 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=4217258 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=343552 peak_demand=136233 required_capacity=1.039375 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=4162769 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=396416 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=4185948 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374272 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=4192830 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=366464 peak_demand=136700 required_capacity=1.042938 + [2026-08-30 05:19:07] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 784.9 | throughput per GPU (TFLOP/s/GPU): 424.8 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101850E+01 | loss scale: 1.0 | grad norm: 45.887 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=143616 expected_recv=262699 expected_overflow=0 exact_recv_per_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=143616 expected_recv=255084 expected_overflow=0 exact_recv_per_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=143616 expected_recv=261278 expected_overflow=0 exact_recv_per_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=143616 expected_recv=262699 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=143616 expected_recv=259028 expected_overflow=0 exact_recv_per_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=143616 expected_recv=255084 actual_valid_recv=257152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=143616 expected_recv=261278 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=143616 expected_recv=259028 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=257152 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=143616 expected_recv=260980 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=143616 expected_recv=257382 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=143616 expected_recv=260980 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=143616 expected_recv=257382 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=143616 expected_recv=254962 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=260096 slack_slots=27136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=143616 expected_recv=254962 actual_valid_recv=257280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=257280 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=143616 expected_recv=256234 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=143616 expected_recv=256234 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=258688 slack_slots=28544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=143616 expected_recv=252763 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264324 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=143616 expected_recv=252763 actual_valid_recv=255488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264324 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=255488 slack_slots=31744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=143616 expected_recv=257021 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=143616 expected_recv=257021 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=143616 expected_recv=263141 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=259584 slack_slots=27648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=143616 expected_recv=263141 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=143616 expected_recv=253159 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=143616 expected_recv=253159 actual_valid_recv=256000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=143616 expected_recv=259863 expected_overflow=0 exact_recv_per_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=256000 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=143616 expected_recv=259863 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=143616 expected_recv=259536 expected_overflow=0 exact_recv_per_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=143616 expected_recv=259181 expected_overflow=0 exact_recv_per_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=143616 expected_recv=259536 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=143616 expected_recv=259181 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=143616 expected_recv=259506 expected_overflow=0 exact_recv_per_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=143616 expected_recv=259506 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=143616 expected_recv=254118 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=143616 expected_recv=258631 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=143616 expected_recv=254118 actual_valid_recv=256640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=143616 expected_recv=258631 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=256640 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=143616 expected_recv=260572 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=260992 slack_slots=26240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=143616 expected_recv=260572 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=268032 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=143616 expected_recv=257160 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=143616 expected_recv=263810 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=143616 expected_recv=259139 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=143616 expected_recv=257160 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=143616 expected_recv=257980 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=143616 expected_recv=263810 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=143616 expected_recv=260026 expected_overflow=0 exact_recv_per_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=143616 expected_recv=259139 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=143616 expected_recv=257980 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=143616 expected_recv=260026 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=260352 slack_slots=26880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=143616 expected_recv=261960 expected_overflow=0 exact_recv_per_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=143616 expected_recv=255121 expected_overflow=0 exact_recv_per_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=143616 expected_recv=261960 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=143616 expected_recv=261234 expected_overflow=0 exact_recv_per_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=143616 expected_recv=255121 actual_valid_recv=257280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=143616 expected_recv=261234 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=257280 slack_slots=29952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=143616 expected_recv=255110 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=143616 expected_recv=255110 actual_valid_recv=258048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=143616 expected_recv=264513 expected_overflow=0 exact_recv_per_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=258048 slack_slots=29184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=143616 expected_recv=264513 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=143616 expected_recv=256572 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=143616 expected_recv=256572 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=143616 expected_recv=261644 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=259328 slack_slots=27904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=143616 expected_recv=259993 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=143616 expected_recv=261644 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=143616 expected_recv=259993 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=143616 expected_recv=262069 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=143616 expected_recv=262069 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=143616 expected_recv=257247 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=143616 expected_recv=257247 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=143616 expected_recv=258778 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=143616 expected_recv=262954 expected_overflow=0 exact_recv_per_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=259968 slack_slots=27264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=143616 expected_recv=258778 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=143616 expected_recv=259198 expected_overflow=0 exact_recv_per_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=143616 expected_recv=262954 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=261120 slack_slots=26112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=265472 slack_slots=21760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=143616 expected_recv=259198 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=143616 expected_recv=262734 expected_overflow=0 exact_recv_per_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=143616 expected_recv=262734 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=265600 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=143616 expected_recv=262103 expected_overflow=0 exact_recv_per_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=143616 expected_recv=261752 expected_overflow=0 exact_recv_per_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=143616 expected_recv=262103 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=143616 expected_recv=264606 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=143616 expected_recv=261752 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=143616 expected_recv=258880 expected_overflow=0 exact_recv_per_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=143616 expected_recv=264606 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=143616 expected_recv=258880 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=143616 expected_recv=258326 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=143616 expected_recv=258326 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=260992 slack_slots=26240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=143616 expected_recv=257943 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=143616 expected_recv=257943 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=259968 slack_slots=27264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=143616 expected_recv=257089 expected_overflow=0 exact_recv_per_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=143616 expected_recv=257211 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=143616 expected_recv=257089 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=143616 expected_recv=257211 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=259968 slack_slots=27264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=143616 expected_recv=262070 expected_overflow=0 exact_recv_per_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=143616 expected_recv=255753 expected_overflow=0 exact_recv_per_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=143616 expected_recv=262070 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=143616 expected_recv=255753 actual_valid_recv=258560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=143616 expected_recv=263180 expected_overflow=0 exact_recv_per_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=143616 expected_recv=256519 expected_overflow=0 exact_recv_per_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=258560 slack_slots=28672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=143616 expected_recv=263180 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=143616 expected_recv=256519 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=265600 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=259456 slack_slots=27776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=143616 expected_recv=255432 expected_overflow=0 exact_recv_per_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=143616 expected_recv=255432 actual_valid_recv=258048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=258048 slack_slots=29184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=143616 expected_recv=261372 expected_overflow=0 exact_recv_per_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=143616 expected_recv=261372 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=143616 expected_recv=252896 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=143616 expected_recv=260603 expected_overflow=0 exact_recv_per_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=143616 expected_recv=252896 actual_valid_recv=256128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=256128 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=143616 expected_recv=260603 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=143616 expected_recv=262157 expected_overflow=0 exact_recv_per_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=143616 expected_recv=255661 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=143616 expected_recv=262157 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=143616 expected_recv=255661 actual_valid_recv=258176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264960 slack_slots=22272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=258176 slack_slots=29056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=143616 expected_recv=253906 expected_overflow=0 exact_recv_per_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=143616 expected_recv=253906 actual_valid_recv=256256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[95502, 133176, 140344, 102742, 151646, 141980, 165539, 117647] avg=131072.0000 max=165539 max_rank=6 min=95502 min_rank=0 imbalance_ratio=1.262962 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=256256 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=143616 expected_recv=261424 expected_overflow=0 exact_recv_per_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[95502, 133176, 140344, 102742, 151646, 141980, 165539, 117647] avg=131072.0000 max=165539 max_rank=6 min=95502 min_rank=0 imbalance_ratio=1.262962 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=143616 expected_recv=261424 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] avg=131072.0000 max=132383 max_rank=1 min=126808 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=33156 max_load_reduction_pct=0.200291 moved_to_replicas=142123 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[95502, 133176, 140344, 102742, 151646, 141980, 165539, 117647] avg=131072.0000 max=165539 max_rank=6 min=95502 min_rank=0 imbalance_ratio=1.262962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] avg=131072.0000 max=132383 max_rank=1 min=126808 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=33156 max_load_reduction_pct=0.200291 moved_to_replicas=142123 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[95502, 133176, 140344, 102742, 151646, 141980, 165539, 117647] avg=131072.0000 max=165539 max_rank=6 min=95502 min_rank=0 imbalance_ratio=1.262962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[95502, 133176, 140344, 102742, 151646, 141980, 165539, 117647] avg=131072.0000 max=165539 max_rank=6 min=95502 min_rank=0 imbalance_ratio=1.262962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[95502, 133176, 140344, 102742, 151646, 141980, 165539, 117647] avg=131072.0000 max=165539 max_rank=6 min=95502 min_rank=0 imbalance_ratio=1.262962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] avg=131072.0000 max=132383 max_rank=1 min=126808 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=33156 max_load_reduction_pct=0.200291 moved_to_replicas=142123 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[142613, 102153, 66376, 151021, 123534, 210388, 100270, 152221] avg=131072.0000 max=210388 max_rank=5 min=66376 min_rank=2 imbalance_ratio=1.605133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[95502, 133176, 140344, 102742, 151646, 141980, 165539, 117647] avg=131072.0000 max=165539 max_rank=6 min=95502 min_rank=0 imbalance_ratio=1.262962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] avg=131072.0000 max=132383 max_rank=1 min=126808 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=33156 max_load_reduction_pct=0.200291 moved_to_replicas=142123 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[95502, 133176, 140344, 102742, 151646, 141980, 165539, 117647] avg=131072.0000 max=165539 max_rank=6 min=95502 min_rank=0 imbalance_ratio=1.262962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] avg=131072.0000 max=132383 max_rank=1 min=126808 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=33156 max_load_reduction_pct=0.200291 moved_to_replicas=142123 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[142613, 102153, 66376, 151021, 123534, 210388, 100270, 152221] avg=131072.0000 max=210388 max_rank=5 min=66376 min_rank=2 imbalance_ratio=1.605133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] avg=131072.0000 max=132383 max_rank=1 min=126808 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=33156 max_load_reduction_pct=0.200291 moved_to_replicas=142123 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=123813 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=78005 max_load_reduction_pct=0.370767 moved_to_replicas=230934 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] avg=131072.0000 max=132383 max_rank=1 min=126808 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=33156 max_load_reduction_pct=0.200291 moved_to_replicas=142123 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[142613, 102153, 66376, 151021, 123534, 210388, 100270, 152221] avg=131072.0000 max=210388 max_rank=5 min=66376 min_rank=2 imbalance_ratio=1.605133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[127471, 132383, 132383, 132383, 132383, 132382, 132383, 126808] avg=131072.0000 max=132383 max_rank=1 min=126808 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=33156 max_load_reduction_pct=0.200291 moved_to_replicas=142123 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=123813 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=78005 max_load_reduction_pct=0.370767 moved_to_replicas=230934 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[142613, 102153, 66376, 151021, 123534, 210388, 100270, 152221] avg=131072.0000 max=210388 max_rank=5 min=66376 min_rank=2 imbalance_ratio=1.605133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[142613, 102153, 66376, 151021, 123534, 210388, 100270, 152221] avg=131072.0000 max=210388 max_rank=5 min=66376 min_rank=2 imbalance_ratio=1.605133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[142613, 102153, 66376, 151021, 123534, 210388, 100270, 152221] avg=131072.0000 max=210388 max_rank=5 min=66376 min_rank=2 imbalance_ratio=1.605133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=123813 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=78005 max_load_reduction_pct=0.370767 moved_to_replicas=230934 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[155350, 178688, 118273, 111691, 147913, 126079, 119078, 91504] avg=131072.0000 max=178688 max_rank=1 min=91504 min_rank=7 imbalance_ratio=1.363281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=123813 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=78005 max_load_reduction_pct=0.370767 moved_to_replicas=230934 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[142613, 102153, 66376, 151021, 123534, 210388, 100270, 152221] avg=131072.0000 max=210388 max_rank=5 min=66376 min_rank=2 imbalance_ratio=1.605133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[155350, 178688, 118273, 111691, 147913, 126079, 119078, 91504] avg=131072.0000 max=178688 max_rank=1 min=91504 min_rank=7 imbalance_ratio=1.363281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[142613, 102153, 66376, 151021, 123534, 210388, 100270, 152221] avg=131072.0000 max=210388 max_rank=5 min=66376 min_rank=2 imbalance_ratio=1.605133 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=123813 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=78005 max_load_reduction_pct=0.370767 moved_to_replicas=230934 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=123813 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=78005 max_load_reduction_pct=0.370767 moved_to_replicas=230934 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] avg=131072.0000 max=132384 max_rank=1 min=126785 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=46304 max_load_reduction_pct=0.259133 moved_to_replicas=170747 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[155350, 178688, 118273, 111691, 147913, 126079, 119078, 91504] avg=131072.0000 max=178688 max_rank=1 min=91504 min_rank=7 imbalance_ratio=1.363281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=123813 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=78005 max_load_reduction_pct=0.370767 moved_to_replicas=230934 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[132383, 132383, 130465, 132383, 123813, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=123813 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=78005 max_load_reduction_pct=0.370767 moved_to_replicas=230934 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] avg=131072.0000 max=132384 max_rank=1 min=126785 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=46304 max_load_reduction_pct=0.259133 moved_to_replicas=170747 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[155350, 178688, 118273, 111691, 147913, 126079, 119078, 91504] avg=131072.0000 max=178688 max_rank=1 min=91504 min_rank=7 imbalance_ratio=1.363281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[155350, 178688, 118273, 111691, 147913, 126079, 119078, 91504] avg=131072.0000 max=178688 max_rank=1 min=91504 min_rank=7 imbalance_ratio=1.363281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[155350, 178688, 118273, 111691, 147913, 126079, 119078, 91504] avg=131072.0000 max=178688 max_rank=1 min=91504 min_rank=7 imbalance_ratio=1.363281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] avg=131072.0000 max=132384 max_rank=1 min=126785 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=46304 max_load_reduction_pct=0.259133 moved_to_replicas=170747 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[188236, 145243, 118754, 129722, 87289, 106785, 171119, 101428] avg=131072.0000 max=188236 max_rank=0 min=87289 min_rank=4 imbalance_ratio=1.436127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] avg=131072.0000 max=132384 max_rank=1 min=126785 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=46304 max_load_reduction_pct=0.259133 moved_to_replicas=170747 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[155350, 178688, 118273, 111691, 147913, 126079, 119078, 91504] avg=131072.0000 max=178688 max_rank=1 min=91504 min_rank=7 imbalance_ratio=1.363281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[155350, 178688, 118273, 111691, 147913, 126079, 119078, 91504] avg=131072.0000 max=178688 max_rank=1 min=91504 min_rank=7 imbalance_ratio=1.363281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] avg=131072.0000 max=132384 max_rank=1 min=126785 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=46304 max_load_reduction_pct=0.259133 moved_to_replicas=170747 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[188236, 145243, 118754, 129722, 87289, 106785, 171119, 101428] avg=131072.0000 max=188236 max_rank=0 min=87289 min_rank=4 imbalance_ratio=1.436127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] avg=131072.0000 max=132384 max_rank=1 min=126785 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=46304 max_load_reduction_pct=0.259133 moved_to_replicas=170747 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] avg=131072.0000 max=132384 max_rank=0 min=126180 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55852 max_load_reduction_pct=0.296713 moved_to_replicas=220121 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[188236, 145243, 118754, 129722, 87289, 106785, 171119, 101428] avg=131072.0000 max=188236 max_rank=0 min=87289 min_rank=4 imbalance_ratio=1.436127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] avg=131072.0000 max=132384 max_rank=1 min=126785 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=46304 max_load_reduction_pct=0.259133 moved_to_replicas=170747 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[132382, 132384, 132384, 126785, 132384, 131940, 127935, 132382] avg=131072.0000 max=132384 max_rank=1 min=126785 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=46304 max_load_reduction_pct=0.259133 moved_to_replicas=170747 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] avg=131072.0000 max=132384 max_rank=0 min=126180 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55852 max_load_reduction_pct=0.296713 moved_to_replicas=220121 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[188236, 145243, 118754, 129722, 87289, 106785, 171119, 101428] avg=131072.0000 max=188236 max_rank=0 min=87289 min_rank=4 imbalance_ratio=1.436127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] avg=131072.0000 max=132384 max_rank=0 min=126180 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55852 max_load_reduction_pct=0.296713 moved_to_replicas=220121 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[81150, 197402, 156125, 114365, 121400, 102046, 174543, 101545] avg=131072.0000 max=197402 max_rank=1 min=81150 min_rank=0 imbalance_ratio=1.506058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[188236, 145243, 118754, 129722, 87289, 106785, 171119, 101428] avg=131072.0000 max=188236 max_rank=0 min=87289 min_rank=4 imbalance_ratio=1.436127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[188236, 145243, 118754, 129722, 87289, 106785, 171119, 101428] avg=131072.0000 max=188236 max_rank=0 min=87289 min_rank=4 imbalance_ratio=1.436127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[188236, 145243, 118754, 129722, 87289, 106785, 171119, 101428] avg=131072.0000 max=188236 max_rank=0 min=87289 min_rank=4 imbalance_ratio=1.436127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] avg=131072.0000 max=132384 max_rank=0 min=126180 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55852 max_load_reduction_pct=0.296713 moved_to_replicas=220121 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[81150, 197402, 156125, 114365, 121400, 102046, 174543, 101545] avg=131072.0000 max=197402 max_rank=1 min=81150 min_rank=0 imbalance_ratio=1.506058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[188236, 145243, 118754, 129722, 87289, 106785, 171119, 101428] avg=131072.0000 max=188236 max_rank=0 min=87289 min_rank=4 imbalance_ratio=1.436127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127124 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65018 max_load_reduction_pct=0.329368 moved_to_replicas=291025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[81150, 197402, 156125, 114365, 121400, 102046, 174543, 101545] avg=131072.0000 max=197402 max_rank=1 min=81150 min_rank=0 imbalance_ratio=1.506058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] avg=131072.0000 max=132384 max_rank=0 min=126180 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55852 max_load_reduction_pct=0.296713 moved_to_replicas=220121 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] avg=131072.0000 max=132384 max_rank=0 min=126180 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55852 max_load_reduction_pct=0.296713 moved_to_replicas=220121 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] avg=131072.0000 max=132384 max_rank=0 min=126180 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55852 max_load_reduction_pct=0.296713 moved_to_replicas=220121 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127124 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65018 max_load_reduction_pct=0.329368 moved_to_replicas=291025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[132384, 132382, 130760, 129722, 126180, 132383, 132382, 132383] avg=131072.0000 max=132384 max_rank=0 min=126180 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=55852 max_load_reduction_pct=0.296713 moved_to_replicas=220121 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[81150, 197402, 156125, 114365, 121400, 102046, 174543, 101545] avg=131072.0000 max=197402 max_rank=1 min=81150 min_rank=0 imbalance_ratio=1.506058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127124 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65018 max_load_reduction_pct=0.329368 moved_to_replicas=291025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[100404, 110548, 116817, 165278, 90519, 214566, 151264, 99180] avg=131072.0000 max=214566 max_rank=5 min=90519 min_rank=4 imbalance_ratio=1.637009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[81150, 197402, 156125, 114365, 121400, 102046, 174543, 101545] avg=131072.0000 max=197402 max_rank=1 min=81150 min_rank=0 imbalance_ratio=1.506058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[81150, 197402, 156125, 114365, 121400, 102046, 174543, 101545] avg=131072.0000 max=197402 max_rank=1 min=81150 min_rank=0 imbalance_ratio=1.506058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[81150, 197402, 156125, 114365, 121400, 102046, 174543, 101545] avg=131072.0000 max=197402 max_rank=1 min=81150 min_rank=0 imbalance_ratio=1.506058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127124 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65018 max_load_reduction_pct=0.329368 moved_to_replicas=291025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[100404, 110548, 116817, 165278, 90519, 214566, 151264, 99180] avg=131072.0000 max=214566 max_rank=5 min=90519 min_rank=4 imbalance_ratio=1.637009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] avg=131072.0000 max=132383 max_rank=0 min=127040 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=82183 max_load_reduction_pct=0.383020 moved_to_replicas=275756 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[81150, 197402, 156125, 114365, 121400, 102046, 174543, 101545] avg=131072.0000 max=197402 max_rank=1 min=81150 min_rank=0 imbalance_ratio=1.506058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[100404, 110548, 116817, 165278, 90519, 214566, 151264, 99180] avg=131072.0000 max=214566 max_rank=5 min=90519 min_rank=4 imbalance_ratio=1.637009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127124 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65018 max_load_reduction_pct=0.329368 moved_to_replicas=291025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127124 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65018 max_load_reduction_pct=0.329368 moved_to_replicas=291025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127124 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65018 max_load_reduction_pct=0.329368 moved_to_replicas=291025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] avg=131072.0000 max=132383 max_rank=0 min=127040 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=82183 max_load_reduction_pct=0.383020 moved_to_replicas=275756 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[127153, 132383, 132383, 132383, 127124, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127124 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65018 max_load_reduction_pct=0.329368 moved_to_replicas=291025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] avg=131072.0000 max=132383 max_rank=0 min=127040 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=82183 max_load_reduction_pct=0.383020 moved_to_replicas=275756 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[100404, 110548, 116817, 165278, 90519, 214566, 151264, 99180] avg=131072.0000 max=214566 max_rank=5 min=90519 min_rank=4 imbalance_ratio=1.637009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[162146, 44220, 142862, 231648, 129779, 132863, 80760, 124298] avg=131072.0000 max=231648 max_rank=3 min=44220 min_rank=1 imbalance_ratio=1.767334 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[100404, 110548, 116817, 165278, 90519, 214566, 151264, 99180] avg=131072.0000 max=214566 max_rank=5 min=90519 min_rank=4 imbalance_ratio=1.637009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[100404, 110548, 116817, 165278, 90519, 214566, 151264, 99180] avg=131072.0000 max=214566 max_rank=5 min=90519 min_rank=4 imbalance_ratio=1.637009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[100404, 110548, 116817, 165278, 90519, 214566, 151264, 99180] avg=131072.0000 max=214566 max_rank=5 min=90519 min_rank=4 imbalance_ratio=1.637009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] avg=131072.0000 max=132383 max_rank=0 min=127040 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=82183 max_load_reduction_pct=0.383020 moved_to_replicas=275756 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] avg=131072.0000 max=132383 max_rank=0 min=124778 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=99265 max_load_reduction_pct=0.428517 moved_to_replicas=314698 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[162146, 44220, 142862, 231648, 129779, 132863, 80760, 124298] avg=131072.0000 max=231648 max_rank=3 min=44220 min_rank=1 imbalance_ratio=1.767334 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[162146, 44220, 142862, 231648, 129779, 132863, 80760, 124298] avg=131072.0000 max=231648 max_rank=3 min=44220 min_rank=1 imbalance_ratio=1.767334 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[100404, 110548, 116817, 165278, 90519, 214566, 151264, 99180] avg=131072.0000 max=214566 max_rank=5 min=90519 min_rank=4 imbalance_ratio=1.637009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] avg=131072.0000 max=132383 max_rank=0 min=127040 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=82183 max_load_reduction_pct=0.383020 moved_to_replicas=275756 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] avg=131072.0000 max=132383 max_rank=0 min=127040 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=82183 max_load_reduction_pct=0.383020 moved_to_replicas=275756 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] avg=131072.0000 max=132383 max_rank=0 min=127040 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=82183 max_load_reduction_pct=0.383020 moved_to_replicas=275756 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] avg=131072.0000 max=132383 max_rank=0 min=124778 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=99265 max_load_reduction_pct=0.428517 moved_to_replicas=314698 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] avg=131072.0000 max=132383 max_rank=0 min=124778 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=99265 max_load_reduction_pct=0.428517 moved_to_replicas=314698 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[132383, 132383, 127040, 132383, 128189, 132383, 132383, 131432] avg=131072.0000 max=132383 max_rank=0 min=127040 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=82183 max_load_reduction_pct=0.383020 moved_to_replicas=275756 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[116143, 135484, 97893, 82929, 108983, 195848, 131153, 180143] avg=131072.0000 max=195848 max_rank=5 min=82929 min_rank=3 imbalance_ratio=1.494202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[162146, 44220, 142862, 231648, 129779, 132863, 80760, 124298] avg=131072.0000 max=231648 max_rank=3 min=44220 min_rank=1 imbalance_ratio=1.767334 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[162146, 44220, 142862, 231648, 129779, 132863, 80760, 124298] avg=131072.0000 max=231648 max_rank=3 min=44220 min_rank=1 imbalance_ratio=1.767334 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[162146, 44220, 142862, 231648, 129779, 132863, 80760, 124298] avg=131072.0000 max=231648 max_rank=3 min=44220 min_rank=1 imbalance_ratio=1.767334 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] avg=131072.0000 max=132384 max_rank=4 min=127642 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63464 max_load_reduction_pct=0.324047 moved_to_replicas=226819 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[162146, 44220, 142862, 231648, 129779, 132863, 80760, 124298] avg=131072.0000 max=231648 max_rank=3 min=44220 min_rank=1 imbalance_ratio=1.767334 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[116143, 135484, 97893, 82929, 108983, 195848, 131153, 180143] avg=131072.0000 max=195848 max_rank=5 min=82929 min_rank=3 imbalance_ratio=1.494202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] avg=131072.0000 max=132383 max_rank=0 min=124778 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=99265 max_load_reduction_pct=0.428517 moved_to_replicas=314698 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[116143, 135484, 97893, 82929, 108983, 195848, 131153, 180143] avg=131072.0000 max=195848 max_rank=5 min=82929 min_rank=3 imbalance_ratio=1.494202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[162146, 44220, 142862, 231648, 129779, 132863, 80760, 124298] avg=131072.0000 max=231648 max_rank=3 min=44220 min_rank=1 imbalance_ratio=1.767334 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] avg=131072.0000 max=132383 max_rank=0 min=124778 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=99265 max_load_reduction_pct=0.428517 moved_to_replicas=314698 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] avg=131072.0000 max=132383 max_rank=0 min=124778 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=99265 max_load_reduction_pct=0.428517 moved_to_replicas=314698 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] avg=131072.0000 max=132384 max_rank=4 min=127642 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63464 max_load_reduction_pct=0.324047 moved_to_replicas=226819 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] avg=131072.0000 max=132383 max_rank=0 min=124778 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=99265 max_load_reduction_pct=0.428517 moved_to_replicas=314698 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] avg=131072.0000 max=132384 max_rank=4 min=127642 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63464 max_load_reduction_pct=0.324047 moved_to_replicas=226819 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129779, 132383, 132105, 124778] avg=131072.0000 max=132383 max_rank=0 min=124778 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=99265 max_load_reduction_pct=0.428517 moved_to_replicas=314698 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[176659, 133804, 94262, 63862, 162067, 119685, 156279, 141958] avg=131072.0000 max=176659 max_rank=0 min=63862 min_rank=3 imbalance_ratio=1.347801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[116143, 135484, 97893, 82929, 108983, 195848, 131153, 180143] avg=131072.0000 max=195848 max_rank=5 min=82929 min_rank=3 imbalance_ratio=1.494202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[116143, 135484, 97893, 82929, 108983, 195848, 131153, 180143] avg=131072.0000 max=195848 max_rank=5 min=82929 min_rank=3 imbalance_ratio=1.494202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[116143, 135484, 97893, 82929, 108983, 195848, 131153, 180143] avg=131072.0000 max=195848 max_rank=5 min=82929 min_rank=3 imbalance_ratio=1.494202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] avg=131072.0000 max=132383 max_rank=1 min=127265 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44276 max_load_reduction_pct=0.250630 moved_to_replicas=220725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[176659, 133804, 94262, 63862, 162067, 119685, 156279, 141958] avg=131072.0000 max=176659 max_rank=0 min=63862 min_rank=3 imbalance_ratio=1.347801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[176659, 133804, 94262, 63862, 162067, 119685, 156279, 141958] avg=131072.0000 max=176659 max_rank=0 min=63862 min_rank=3 imbalance_ratio=1.347801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] avg=131072.0000 max=132384 max_rank=4 min=127642 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63464 max_load_reduction_pct=0.324047 moved_to_replicas=226819 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[116143, 135484, 97893, 82929, 108983, 195848, 131153, 180143] avg=131072.0000 max=195848 max_rank=5 min=82929 min_rank=3 imbalance_ratio=1.494202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] avg=131072.0000 max=132384 max_rank=4 min=127642 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63464 max_load_reduction_pct=0.324047 moved_to_replicas=226819 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[116143, 135484, 97893, 82929, 108983, 195848, 131153, 180143] avg=131072.0000 max=195848 max_rank=5 min=82929 min_rank=3 imbalance_ratio=1.494202 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] avg=131072.0000 max=132384 max_rank=4 min=127642 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63464 max_load_reduction_pct=0.324047 moved_to_replicas=226819 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] avg=131072.0000 max=132383 max_rank=1 min=127265 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44276 max_load_reduction_pct=0.250630 moved_to_replicas=220725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] avg=131072.0000 max=132383 max_rank=1 min=127265 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44276 max_load_reduction_pct=0.250630 moved_to_replicas=220725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] avg=131072.0000 max=132384 max_rank=4 min=127642 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63464 max_load_reduction_pct=0.324047 moved_to_replicas=226819 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[133995, 160219, 136435, 102991, 184069, 142802, 76187, 111878] avg=131072.0000 max=184069 max_rank=4 min=76187 min_rank=6 imbalance_ratio=1.404335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[176659, 133804, 94262, 63862, 162067, 119685, 156279, 141958] avg=131072.0000 max=176659 max_rank=0 min=63862 min_rank=3 imbalance_ratio=1.347801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[131533, 132383, 128715, 127642, 132384, 132383, 131153, 132383] avg=131072.0000 max=132384 max_rank=4 min=127642 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63464 max_load_reduction_pct=0.324047 moved_to_replicas=226819 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] avg=131072.0000 max=132384 max_rank=5 min=126665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=51685 max_load_reduction_pct=0.280791 moved_to_replicas=197601 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[176659, 133804, 94262, 63862, 162067, 119685, 156279, 141958] avg=131072.0000 max=176659 max_rank=0 min=63862 min_rank=3 imbalance_ratio=1.347801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[176659, 133804, 94262, 63862, 162067, 119685, 156279, 141958] avg=131072.0000 max=176659 max_rank=0 min=63862 min_rank=3 imbalance_ratio=1.347801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[133995, 160219, 136435, 102991, 184069, 142802, 76187, 111878] avg=131072.0000 max=184069 max_rank=4 min=76187 min_rank=6 imbalance_ratio=1.404335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[133995, 160219, 136435, 102991, 184069, 142802, 76187, 111878] avg=131072.0000 max=184069 max_rank=4 min=76187 min_rank=6 imbalance_ratio=1.404335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[176659, 133804, 94262, 63862, 162067, 119685, 156279, 141958] avg=131072.0000 max=176659 max_rank=0 min=63862 min_rank=3 imbalance_ratio=1.347801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] avg=131072.0000 max=132383 max_rank=1 min=127265 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44276 max_load_reduction_pct=0.250630 moved_to_replicas=220725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] avg=131072.0000 max=132383 max_rank=1 min=127265 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44276 max_load_reduction_pct=0.250630 moved_to_replicas=220725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] avg=131072.0000 max=132383 max_rank=1 min=127265 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44276 max_load_reduction_pct=0.250630 moved_to_replicas=220725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[176659, 133804, 94262, 63862, 162067, 119685, 156279, 141958] avg=131072.0000 max=176659 max_rank=0 min=63862 min_rank=3 imbalance_ratio=1.347801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] avg=131072.0000 max=132384 max_rank=5 min=126665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=51685 max_load_reduction_pct=0.280791 moved_to_replicas=197601 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[41964, 138573, 157299, 135577, 103892, 123528, 129369, 218374] avg=131072.0000 max=218374 max_rank=7 min=41964 min_rank=0 imbalance_ratio=1.666061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] avg=131072.0000 max=132384 max_rank=5 min=126665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=51685 max_load_reduction_pct=0.280791 moved_to_replicas=197601 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] avg=131072.0000 max=132383 max_rank=1 min=127265 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44276 max_load_reduction_pct=0.250630 moved_to_replicas=220725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[133995, 160219, 136435, 102991, 184069, 142802, 76187, 111878] avg=131072.0000 max=184069 max_rank=4 min=76187 min_rank=6 imbalance_ratio=1.404335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130138, 127265, 132383, 129260, 132382, 132383] avg=131072.0000 max=132383 max_rank=1 min=127265 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44276 max_load_reduction_pct=0.250630 moved_to_replicas=220725 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] avg=131072.0000 max=132383 max_rank=2 min=129369 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=85991 max_load_reduction_pct=0.393779 moved_to_replicas=265001 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[133995, 160219, 136435, 102991, 184069, 142802, 76187, 111878] avg=131072.0000 max=184069 max_rank=4 min=76187 min_rank=6 imbalance_ratio=1.404335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[133995, 160219, 136435, 102991, 184069, 142802, 76187, 111878] avg=131072.0000 max=184069 max_rank=4 min=76187 min_rank=6 imbalance_ratio=1.404335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[41964, 138573, 157299, 135577, 103892, 123528, 129369, 218374] avg=131072.0000 max=218374 max_rank=7 min=41964 min_rank=0 imbalance_ratio=1.666061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[41964, 138573, 157299, 135577, 103892, 123528, 129369, 218374] avg=131072.0000 max=218374 max_rank=7 min=41964 min_rank=0 imbalance_ratio=1.666061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[133995, 160219, 136435, 102991, 184069, 142802, 76187, 111878] avg=131072.0000 max=184069 max_rank=4 min=76187 min_rank=6 imbalance_ratio=1.404335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] avg=131072.0000 max=132384 max_rank=5 min=126665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=51685 max_load_reduction_pct=0.280791 moved_to_replicas=197601 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] avg=131072.0000 max=132384 max_rank=5 min=126665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=51685 max_load_reduction_pct=0.280791 moved_to_replicas=197601 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] avg=131072.0000 max=132384 max_rank=5 min=126665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=51685 max_load_reduction_pct=0.280791 moved_to_replicas=197601 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[108413, 152745, 113241, 110369, 91443, 190284, 133329, 148752] avg=131072.0000 max=190284 max_rank=5 min=91443 min_rank=4 imbalance_ratio=1.451752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] avg=131072.0000 max=132383 max_rank=2 min=129369 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=85991 max_load_reduction_pct=0.393779 moved_to_replicas=265001 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[133995, 160219, 136435, 102991, 184069, 142802, 76187, 111878] avg=131072.0000 max=184069 max_rank=4 min=76187 min_rank=6 imbalance_ratio=1.404335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] avg=131072.0000 max=132383 max_rank=2 min=129369 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=85991 max_load_reduction_pct=0.393779 moved_to_replicas=265001 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] avg=131072.0000 max=132384 max_rank=5 min=126665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=51685 max_load_reduction_pct=0.280791 moved_to_replicas=197601 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=124828 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57900 max_load_reduction_pct=0.304282 moved_to_replicas=197403 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[41964, 138573, 157299, 135577, 103892, 123528, 129369, 218374] avg=131072.0000 max=218374 max_rank=7 min=41964 min_rank=0 imbalance_ratio=1.666061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127611, 132383, 132384, 132384, 126665] avg=131072.0000 max=132384 max_rank=5 min=126665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=51685 max_load_reduction_pct=0.280791 moved_to_replicas=197601 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[41964, 138573, 157299, 135577, 103892, 123528, 129369, 218374] avg=131072.0000 max=218374 max_rank=7 min=41964 min_rank=0 imbalance_ratio=1.666061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[41964, 138573, 157299, 135577, 103892, 123528, 129369, 218374] avg=131072.0000 max=218374 max_rank=7 min=41964 min_rank=0 imbalance_ratio=1.666061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[108413, 152745, 113241, 110369, 91443, 190284, 133329, 148752] avg=131072.0000 max=190284 max_rank=5 min=91443 min_rank=4 imbalance_ratio=1.451752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[41964, 138573, 157299, 135577, 103892, 123528, 129369, 218374] avg=131072.0000 max=218374 max_rank=7 min=41964 min_rank=0 imbalance_ratio=1.666061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] avg=131072.0000 max=132383 max_rank=2 min=129369 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=85991 max_load_reduction_pct=0.393779 moved_to_replicas=265001 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[108413, 152745, 113241, 110369, 91443, 190284, 133329, 148752] avg=131072.0000 max=190284 max_rank=5 min=91443 min_rank=4 imbalance_ratio=1.451752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[140418, 160546, 130521, 133629, 142815, 70647, 130797, 139203] avg=131072.0000 max=160546 max_rank=1 min=70647 min_rank=5 imbalance_ratio=1.224869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] avg=131072.0000 max=132383 max_rank=2 min=129369 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=85991 max_load_reduction_pct=0.393779 moved_to_replicas=265001 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] avg=131072.0000 max=132383 max_rank=2 min=129369 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=85991 max_load_reduction_pct=0.393779 moved_to_replicas=265001 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=124828 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57900 max_load_reduction_pct=0.304282 moved_to_replicas=197403 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[41964, 138573, 157299, 135577, 103892, 123528, 129369, 218374] avg=131072.0000 max=218374 max_rank=7 min=41964 min_rank=0 imbalance_ratio=1.666061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] avg=131072.0000 max=132383 max_rank=2 min=129369 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=85991 max_load_reduction_pct=0.393779 moved_to_replicas=265001 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=124828 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57900 max_load_reduction_pct=0.304282 moved_to_replicas=197403 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] avg=131072.0000 max=132384 max_rank=4 min=125345 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28162 max_load_reduction_pct=0.175414 moved_to_replicas=115887 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[108413, 152745, 113241, 110369, 91443, 190284, 133329, 148752] avg=131072.0000 max=190284 max_rank=5 min=91443 min_rank=4 imbalance_ratio=1.451752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[130351, 132382, 132383, 132383, 129607, 129719, 129369, 132382] avg=131072.0000 max=132383 max_rank=2 min=129369 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=85991 max_load_reduction_pct=0.393779 moved_to_replicas=265001 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[108413, 152745, 113241, 110369, 91443, 190284, 133329, 148752] avg=131072.0000 max=190284 max_rank=5 min=91443 min_rank=4 imbalance_ratio=1.451752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[140418, 160546, 130521, 133629, 142815, 70647, 130797, 139203] avg=131072.0000 max=160546 max_rank=1 min=70647 min_rank=5 imbalance_ratio=1.224869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[108413, 152745, 113241, 110369, 91443, 190284, 133329, 148752] avg=131072.0000 max=190284 max_rank=5 min=91443 min_rank=4 imbalance_ratio=1.451752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[108413, 152745, 113241, 110369, 91443, 190284, 133329, 148752] avg=131072.0000 max=190284 max_rank=5 min=91443 min_rank=4 imbalance_ratio=1.451752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=124828 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57900 max_load_reduction_pct=0.304282 moved_to_replicas=197403 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[140418, 160546, 130521, 133629, 142815, 70647, 130797, 139203] avg=131072.0000 max=160546 max_rank=1 min=70647 min_rank=5 imbalance_ratio=1.224869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[88531, 45953, 158984, 138133, 133686, 188381, 132808, 162100] avg=131072.0000 max=188381 max_rank=5 min=45953 min_rank=1 imbalance_ratio=1.437233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=124828 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57900 max_load_reduction_pct=0.304282 moved_to_replicas=197403 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] avg=131072.0000 max=132384 max_rank=4 min=125345 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28162 max_load_reduction_pct=0.175414 moved_to_replicas=115887 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[108413, 152745, 113241, 110369, 91443, 190284, 133329, 148752] avg=131072.0000 max=190284 max_rank=5 min=91443 min_rank=4 imbalance_ratio=1.451752 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=124828 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57900 max_load_reduction_pct=0.304282 moved_to_replicas=197403 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=124828 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57900 max_load_reduction_pct=0.304282 moved_to_replicas=197403 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] avg=131072.0000 max=132384 max_rank=4 min=125345 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28162 max_load_reduction_pct=0.175414 moved_to_replicas=115887 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=125726 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=55998 max_load_reduction_pct=0.297259 moved_to_replicas=243944 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[140418, 160546, 130521, 133629, 142815, 70647, 130797, 139203] avg=131072.0000 max=160546 max_rank=1 min=70647 min_rank=5 imbalance_ratio=1.224869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[140418, 160546, 130521, 133629, 142815, 70647, 130797, 139203] avg=131072.0000 max=160546 max_rank=1 min=70647 min_rank=5 imbalance_ratio=1.224869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132383, 129609, 132223, 124828, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=124828 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57900 max_load_reduction_pct=0.304282 moved_to_replicas=197403 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[88531, 45953, 158984, 138133, 133686, 188381, 132808, 162100] avg=131072.0000 max=188381 max_rank=5 min=45953 min_rank=1 imbalance_ratio=1.437233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[140418, 160546, 130521, 133629, 142815, 70647, 130797, 139203] avg=131072.0000 max=160546 max_rank=1 min=70647 min_rank=5 imbalance_ratio=1.224869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[88531, 45953, 158984, 138133, 133686, 188381, 132808, 162100] avg=131072.0000 max=188381 max_rank=5 min=45953 min_rank=1 imbalance_ratio=1.437233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] avg=131072.0000 max=132384 max_rank=4 min=125345 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28162 max_load_reduction_pct=0.175414 moved_to_replicas=115887 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[171407, 159168, 25812, 102842, 115979, 184039, 141350, 147979] avg=131072.0000 max=184039 max_rank=5 min=25812 min_rank=2 imbalance_ratio=1.404106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] avg=131072.0000 max=132384 max_rank=4 min=125345 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28162 max_load_reduction_pct=0.175414 moved_to_replicas=115887 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[140418, 160546, 130521, 133629, 142815, 70647, 130797, 139203] avg=131072.0000 max=160546 max_rank=1 min=70647 min_rank=5 imbalance_ratio=1.224869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=125726 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=55998 max_load_reduction_pct=0.297259 moved_to_replicas=243944 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] avg=131072.0000 max=132384 max_rank=4 min=125345 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28162 max_load_reduction_pct=0.175414 moved_to_replicas=115887 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[140418, 160546, 130521, 133629, 142815, 70647, 130797, 139203] avg=131072.0000 max=160546 max_rank=1 min=70647 min_rank=5 imbalance_ratio=1.224869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=125726 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=55998 max_load_reduction_pct=0.297259 moved_to_replicas=243944 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] avg=131072.0000 max=132385 max_rank=3 min=124947 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=51654 max_load_reduction_pct=0.280669 moved_to_replicas=311335 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] avg=131072.0000 max=132384 max_rank=4 min=125345 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28162 max_load_reduction_pct=0.175414 moved_to_replicas=115887 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[88531, 45953, 158984, 138133, 133686, 188381, 132808, 162100] avg=131072.0000 max=188381 max_rank=5 min=45953 min_rank=1 imbalance_ratio=1.437233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[88531, 45953, 158984, 138133, 133686, 188381, 132808, 162100] avg=131072.0000 max=188381 max_rank=5 min=45953 min_rank=1 imbalance_ratio=1.437233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[171407, 159168, 25812, 102842, 115979, 184039, 141350, 147979] avg=131072.0000 max=184039 max_rank=5 min=25812 min_rank=2 imbalance_ratio=1.404106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[132382, 132381, 130521, 132383, 132384, 125345, 130797, 132383] avg=131072.0000 max=132384 max_rank=4 min=125345 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28162 max_load_reduction_pct=0.175414 moved_to_replicas=115887 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[88531, 45953, 158984, 138133, 133686, 188381, 132808, 162100] avg=131072.0000 max=188381 max_rank=5 min=45953 min_rank=1 imbalance_ratio=1.437233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=125726 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=55998 max_load_reduction_pct=0.297259 moved_to_replicas=243944 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[171407, 159168, 25812, 102842, 115979, 184039, 141350, 147979] avg=131072.0000 max=184039 max_rank=5 min=25812 min_rank=2 imbalance_ratio=1.404106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=125726 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=55998 max_load_reduction_pct=0.297259 moved_to_replicas=243944 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[115961, 97776, 117902, 106347, 102827, 191531, 145560, 170672] avg=131072.0000 max=191531 max_rank=5 min=97776 min_rank=1 imbalance_ratio=1.461266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[88531, 45953, 158984, 138133, 133686, 188381, 132808, 162100] avg=131072.0000 max=188381 max_rank=5 min=45953 min_rank=1 imbalance_ratio=1.437233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] avg=131072.0000 max=132385 max_rank=3 min=124947 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=51654 max_load_reduction_pct=0.280669 moved_to_replicas=311335 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=125726 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=55998 max_load_reduction_pct=0.297259 moved_to_replicas=243944 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] avg=131072.0000 max=132385 max_rank=3 min=124947 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=51654 max_load_reduction_pct=0.280669 moved_to_replicas=311335 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[88531, 45953, 158984, 138133, 133686, 188381, 132808, 162100] avg=131072.0000 max=188381 max_rank=5 min=45953 min_rank=1 imbalance_ratio=1.437233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126910 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=59147 max_load_reduction_pct=0.308812 moved_to_replicas=219800 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[171407, 159168, 25812, 102842, 115979, 184039, 141350, 147979] avg=131072.0000 max=184039 max_rank=5 min=25812 min_rank=2 imbalance_ratio=1.404106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=125726 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=55998 max_load_reduction_pct=0.297259 moved_to_replicas=243944 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[171407, 159168, 25812, 102842, 115979, 184039, 141350, 147979] avg=131072.0000 max=184039 max_rank=5 min=25812 min_rank=2 imbalance_ratio=1.404106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[115961, 97776, 117902, 106347, 102827, 191531, 145560, 170672] avg=131072.0000 max=191531 max_rank=5 min=97776 min_rank=1 imbalance_ratio=1.461266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[125726, 128554, 132383, 132383, 132383, 132381, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=125726 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=55998 max_load_reduction_pct=0.297259 moved_to_replicas=243944 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[171407, 159168, 25812, 102842, 115979, 184039, 141350, 147979] avg=131072.0000 max=184039 max_rank=5 min=25812 min_rank=2 imbalance_ratio=1.404106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[115961, 97776, 117902, 106347, 102827, 191531, 145560, 170672] avg=131072.0000 max=191531 max_rank=5 min=97776 min_rank=1 imbalance_ratio=1.461266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] avg=131072.0000 max=132385 max_rank=3 min=124947 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=51654 max_load_reduction_pct=0.280669 moved_to_replicas=311335 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] avg=131072.0000 max=132385 max_rank=3 min=124947 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=51654 max_load_reduction_pct=0.280669 moved_to_replicas=311335 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126910 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=59147 max_load_reduction_pct=0.308812 moved_to_replicas=219800 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[171407, 159168, 25812, 102842, 115979, 184039, 141350, 147979] avg=131072.0000 max=184039 max_rank=5 min=25812 min_rank=2 imbalance_ratio=1.404106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] avg=131072.0000 max=132385 max_rank=3 min=124947 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=51654 max_load_reduction_pct=0.280669 moved_to_replicas=311335 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126910 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=59147 max_load_reduction_pct=0.308812 moved_to_replicas=219800 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[171407, 159168, 25812, 102842, 115979, 184039, 141350, 147979] avg=131072.0000 max=184039 max_rank=5 min=25812 min_rank=2 imbalance_ratio=1.404106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[115961, 97776, 117902, 106347, 102827, 191531, 145560, 170672] avg=131072.0000 max=191531 max_rank=5 min=97776 min_rank=1 imbalance_ratio=1.461266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] avg=131072.0000 max=132385 max_rank=3 min=124947 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=51654 max_load_reduction_pct=0.280669 moved_to_replicas=311335 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[115961, 97776, 117902, 106347, 102827, 191531, 145560, 170672] avg=131072.0000 max=191531 max_rank=5 min=97776 min_rank=1 imbalance_ratio=1.461266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[132382, 132382, 129333, 132385, 124947, 132382, 132382, 132383] avg=131072.0000 max=132385 max_rank=3 min=124947 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=51654 max_load_reduction_pct=0.280669 moved_to_replicas=311335 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[115961, 97776, 117902, 106347, 102827, 191531, 145560, 170672] avg=131072.0000 max=191531 max_rank=5 min=97776 min_rank=1 imbalance_ratio=1.461266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126910 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=59147 max_load_reduction_pct=0.308812 moved_to_replicas=219800 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126910 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=59147 max_load_reduction_pct=0.308812 moved_to_replicas=219800 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[115961, 97776, 117902, 106347, 102827, 191531, 145560, 170672] avg=131072.0000 max=191531 max_rank=5 min=97776 min_rank=1 imbalance_ratio=1.461266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126910 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=59147 max_load_reduction_pct=0.308812 moved_to_replicas=219800 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[115961, 97776, 117902, 106347, 102827, 191531, 145560, 170672] avg=131072.0000 max=191531 max_rank=5 min=97776 min_rank=1 imbalance_ratio=1.461266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126910 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=59147 max_load_reduction_pct=0.308812 moved_to_replicas=219800 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[132384, 130121, 126910, 132382, 129632, 132381, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126910 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=59147 max_load_reduction_pct=0.308812 moved_to_replicas=219800 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=4219963 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=336896 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=4176406 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=378240 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=4203675 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=353920 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=4204387 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=351872 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=4198030 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=359040 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=4209396 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=348928 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=4172465 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=384768 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=4170110 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=386816 peak_demand=132384 required_capacity=1.010010 + [2026-08-30 05:19:08] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 564.9 | throughput per GPU (TFLOP/s/GPU): 590.1 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.060052E+01 | loss scale: 1.0 | grad norm: 216.435 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=143616 expected_recv=260227 expected_overflow=0 exact_recv_per_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=143616 expected_recv=259851 expected_overflow=0 exact_recv_per_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=143616 expected_recv=256670 expected_overflow=0 exact_recv_per_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=143616 expected_recv=261340 expected_overflow=0 exact_recv_per_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=143616 expected_recv=260227 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=143616 expected_recv=259851 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=143616 expected_recv=256670 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=143616 expected_recv=261340 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=143616 expected_recv=250657 expected_overflow=0 exact_recv_per_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=143616 expected_recv=261181 expected_overflow=0 exact_recv_per_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=143616 expected_recv=250657 actual_valid_recv=253056 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=143616 expected_recv=261181 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=143616 expected_recv=261485 expected_overflow=0 exact_recv_per_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=253056 slack_slots=34176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=143616 expected_recv=261485 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=143616 expected_recv=264444 expected_overflow=0 exact_recv_per_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=143616 expected_recv=259627 expected_overflow=0 exact_recv_per_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=143616 expected_recv=253669 expected_overflow=0 exact_recv_per_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=143616 expected_recv=264444 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=143616 expected_recv=260348 expected_overflow=0 exact_recv_per_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=143616 expected_recv=259627 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=143616 expected_recv=253669 actual_valid_recv=257024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=143616 expected_recv=260348 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=257024 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=143616 expected_recv=252422 expected_overflow=0 exact_recv_per_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=143616 expected_recv=261872 expected_overflow=0 exact_recv_per_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=143616 expected_recv=252422 actual_valid_recv=255360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=143616 expected_recv=261872 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=143616 expected_recv=259032 expected_overflow=0 exact_recv_per_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=255360 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=264960 slack_slots=22272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=143616 expected_recv=259032 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=143616 expected_recv=266785 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=143616 expected_recv=266786 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=143616 expected_recv=266785 actual_valid_recv=269184 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=143616 expected_recv=266786 actual_valid_recv=269568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=269184 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=269568 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=143616 expected_recv=266787 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=143616 expected_recv=266785 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=143616 expected_recv=264965 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=143616 expected_recv=255855 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=143616 expected_recv=242403 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=143616 expected_recv=266787 actual_valid_recv=269568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=143616 expected_recv=266785 actual_valid_recv=269568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=143616 expected_recv=266786 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=143616 expected_recv=264965 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=143616 expected_recv=255855 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=143616 expected_recv=242403 actual_valid_recv=245248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=269568 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=269568 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=143616 expected_recv=266786 actual_valid_recv=269440 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=245248 slack_slots=41984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=269440 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=143616 expected_recv=262202 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=143616 expected_recv=262202 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=143616 expected_recv=250966 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=264832 slack_slots=22400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=143616 expected_recv=260154 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=143616 expected_recv=250966 actual_valid_recv=253952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=253952 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=143616 expected_recv=260154 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263424 slack_slots=23808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=143616 expected_recv=262795 expected_overflow=0 exact_recv_per_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=143616 expected_recv=262925 expected_overflow=0 exact_recv_per_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=143616 expected_recv=262795 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=143616 expected_recv=262925 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=143616 expected_recv=263144 expected_overflow=0 exact_recv_per_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=265600 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=143616 expected_recv=263144 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=143616 expected_recv=254905 expected_overflow=0 exact_recv_per_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=143616 expected_recv=259084 expected_overflow=0 exact_recv_per_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=143616 expected_recv=254905 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=143616 expected_recv=259084 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=257792 slack_slots=29440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=143616 expected_recv=262622 expected_overflow=0 exact_recv_per_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=143616 expected_recv=258834 expected_overflow=0 exact_recv_per_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=143616 expected_recv=262622 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=143616 expected_recv=258834 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=265344 slack_slots=21888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=143616 expected_recv=254941 expected_overflow=0 exact_recv_per_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=143616 expected_recv=261688 expected_overflow=0 exact_recv_per_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=143616 expected_recv=267400 expected_overflow=0 exact_recv_per_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=143616 expected_recv=254941 actual_valid_recv=257664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=143616 expected_recv=261688 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=143616 expected_recv=265847 expected_overflow=0 exact_recv_per_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=143616 expected_recv=267400 actual_valid_recv=270208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=257664 slack_slots=29568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=143616 expected_recv=265847 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=143616 expected_recv=253003 expected_overflow=0 exact_recv_per_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=270208 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=143616 expected_recv=249967 expected_overflow=0 exact_recv_per_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=268032 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=143616 expected_recv=253003 actual_valid_recv=255360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=143616 expected_recv=249967 actual_valid_recv=252672 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=143616 expected_recv=267399 expected_overflow=0 exact_recv_per_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=255360 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=252672 slack_slots=34560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=143616 expected_recv=267399 expected_overflow=0 exact_recv_per_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=143616 expected_recv=267399 actual_valid_recv=269568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=143616 expected_recv=267399 actual_valid_recv=270208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=269568 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=143616 expected_recv=258734 expected_overflow=0 exact_recv_per_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=143616 expected_recv=267403 expected_overflow=0 exact_recv_per_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=143616 expected_recv=262118 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=270208 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=143616 expected_recv=267403 actual_valid_recv=270592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=143616 expected_recv=258734 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=143616 expected_recv=262118 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=270592 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=143616 expected_recv=263421 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264832 slack_slots=22400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=143616 expected_recv=255122 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=143616 expected_recv=263421 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=143616 expected_recv=255122 actual_valid_recv=258560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=258560 slack_slots=28672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=143616 expected_recv=257428 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=143616 expected_recv=257428 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=143616 expected_recv=255654 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=260352 slack_slots=26880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=143616 expected_recv=255654 actual_valid_recv=258432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=258432 slack_slots=28800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264092 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=143616 expected_recv=253575 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264092 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=143616 expected_recv=254399 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=143616 expected_recv=253575 actual_valid_recv=256384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=268160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=143616 expected_recv=254399 actual_valid_recv=257024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=256384 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=268160 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=257024 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=143616 expected_recv=264170 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=143616 expected_recv=262903 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=143616 expected_recv=261801 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=143616 expected_recv=264170 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=143616 expected_recv=261901 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=143616 expected_recv=262903 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=143616 expected_recv=252636 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=143616 expected_recv=261801 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=265600 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=143616 expected_recv=262445 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=143616 expected_recv=261901 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=143616 expected_recv=252636 actual_valid_recv=255232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=143616 expected_recv=262445 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=255232 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=265216 slack_slots=22016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=143616 expected_recv=264321 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=143616 expected_recv=262385 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=143616 expected_recv=264321 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=143616 expected_recv=255643 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=143616 expected_recv=262385 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=143616 expected_recv=258745 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=143616 expected_recv=255643 actual_valid_recv=258432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=258432 slack_slots=28800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=143616 expected_recv=258745 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=143616 expected_recv=263161 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=143616 expected_recv=263161 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=143616 expected_recv=263773 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=143616 expected_recv=263692 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=143616 expected_recv=263773 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=143616 expected_recv=263692 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=143616 expected_recv=256587 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=143616 expected_recv=259988 expected_overflow=0 exact_recv_per_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=143616 expected_recv=256587 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=143616 expected_recv=259988 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=259456 slack_slots=27776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=143616 expected_recv=259698 expected_overflow=0 exact_recv_per_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=143616 expected_recv=259698 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=143616 expected_recv=253633 expected_overflow=0 exact_recv_per_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=262912 slack_slots=24320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=143616 expected_recv=253633 actual_valid_recv=256640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=143616 expected_recv=259162 expected_overflow=0 exact_recv_per_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=256640 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=143616 expected_recv=259162 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=143616 expected_recv=261824 expected_overflow=0 exact_recv_per_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=143616 expected_recv=261824 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=143616 expected_recv=258929 expected_overflow=0 exact_recv_per_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264960 slack_slots=22272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=143616 expected_recv=258929 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[91983, 127844, 142019, 106941, 154021, 143839, 168476, 113453] avg=131072.0000 max=168476 max_rank=6 min=91983 min_rank=0 imbalance_ratio=1.285370 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=143616 expected_recv=258170 expected_overflow=0 exact_recv_per_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[91983, 127844, 142019, 106941, 154021, 143839, 168476, 113453] avg=131072.0000 max=168476 max_rank=6 min=91983 min_rank=0 imbalance_ratio=1.285370 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=143616 expected_recv=258170 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[91983, 127844, 142019, 106941, 154021, 143839, 168476, 113453] avg=131072.0000 max=168476 max_rank=6 min=91983 min_rank=0 imbalance_ratio=1.285370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127844 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36092 max_load_reduction_pct=0.214226 moved_to_replicas=155306 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[91983, 127844, 142019, 106941, 154021, 143839, 168476, 113453] avg=131072.0000 max=168476 max_rank=6 min=91983 min_rank=0 imbalance_ratio=1.285370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127844 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36092 max_load_reduction_pct=0.214226 moved_to_replicas=155306 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[91983, 127844, 142019, 106941, 154021, 143839, 168476, 113453] avg=131072.0000 max=168476 max_rank=6 min=91983 min_rank=0 imbalance_ratio=1.285370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127844 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36092 max_load_reduction_pct=0.214226 moved_to_replicas=155306 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[91983, 127844, 142019, 106941, 154021, 143839, 168476, 113453] avg=131072.0000 max=168476 max_rank=6 min=91983 min_rank=0 imbalance_ratio=1.285370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[91983, 127844, 142019, 106941, 154021, 143839, 168476, 113453] avg=131072.0000 max=168476 max_rank=6 min=91983 min_rank=0 imbalance_ratio=1.285370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127844 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36092 max_load_reduction_pct=0.214226 moved_to_replicas=155306 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[77808, 147051, 89557, 125378, 240126, 151212, 77649, 139795] avg=131072.0000 max=240126 max_rank=4 min=77649 min_rank=6 imbalance_ratio=1.832016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[91983, 127844, 142019, 106941, 154021, 143839, 168476, 113453] avg=131072.0000 max=168476 max_rank=6 min=91983 min_rank=0 imbalance_ratio=1.285370 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127844 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36092 max_load_reduction_pct=0.214226 moved_to_replicas=155306 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[77808, 147051, 89557, 125378, 240126, 151212, 77649, 139795] avg=131072.0000 max=240126 max_rank=4 min=77649 min_rank=6 imbalance_ratio=1.832016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127844 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36092 max_load_reduction_pct=0.214226 moved_to_replicas=155306 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127844 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36092 max_load_reduction_pct=0.214226 moved_to_replicas=155306 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[77808, 147051, 89557, 125378, 240126, 151212, 77649, 139795] avg=131072.0000 max=240126 max_rank=4 min=77649 min_rank=6 imbalance_ratio=1.832016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] avg=131072.0000 max=132384 max_rank=1 min=125378 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107742 max_load_reduction_pct=0.448689 moved_to_replicas=299136 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[77808, 147051, 89557, 125378, 240126, 151212, 77649, 139795] avg=131072.0000 max=240126 max_rank=4 min=77649 min_rank=6 imbalance_ratio=1.832016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[129831, 127844, 132382, 128987, 132383, 132382, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=127844 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36092 max_load_reduction_pct=0.214226 moved_to_replicas=155306 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] avg=131072.0000 max=132384 max_rank=1 min=125378 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107742 max_load_reduction_pct=0.448689 moved_to_replicas=299136 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] avg=131072.0000 max=132384 max_rank=1 min=125378 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107742 max_load_reduction_pct=0.448689 moved_to_replicas=299136 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[77808, 147051, 89557, 125378, 240126, 151212, 77649, 139795] avg=131072.0000 max=240126 max_rank=4 min=77649 min_rank=6 imbalance_ratio=1.832016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[77808, 147051, 89557, 125378, 240126, 151212, 77649, 139795] avg=131072.0000 max=240126 max_rank=4 min=77649 min_rank=6 imbalance_ratio=1.832016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[77808, 147051, 89557, 125378, 240126, 151212, 77649, 139795] avg=131072.0000 max=240126 max_rank=4 min=77649 min_rank=6 imbalance_ratio=1.832016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] avg=131072.0000 max=132384 max_rank=1 min=125378 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107742 max_load_reduction_pct=0.448689 moved_to_replicas=299136 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[123715, 208605, 135911, 113132, 168736, 108695, 146957, 42825] avg=131072.0000 max=208605 max_rank=1 min=42825 min_rank=7 imbalance_ratio=1.591530 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[77808, 147051, 89557, 125378, 240126, 151212, 77649, 139795] avg=131072.0000 max=240126 max_rank=4 min=77649 min_rank=6 imbalance_ratio=1.832016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[123715, 208605, 135911, 113132, 168736, 108695, 146957, 42825] avg=131072.0000 max=208605 max_rank=1 min=42825 min_rank=7 imbalance_ratio=1.591530 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] avg=131072.0000 max=132384 max_rank=1 min=125378 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107742 max_load_reduction_pct=0.448689 moved_to_replicas=299136 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] avg=131072.0000 max=132384 max_rank=1 min=125378 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107742 max_load_reduction_pct=0.448689 moved_to_replicas=299136 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] avg=131072.0000 max=132384 max_rank=1 min=125378 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107742 max_load_reduction_pct=0.448689 moved_to_replicas=299136 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] avg=131072.0000 max=132384 max_rank=6 min=127034 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=76221 max_load_reduction_pct=0.365384 moved_to_replicas=256296 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[123715, 208605, 135911, 113132, 168736, 108695, 146957, 42825] avg=131072.0000 max=208605 max_rank=1 min=42825 min_rank=7 imbalance_ratio=1.591530 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[123715, 208605, 135911, 113132, 168736, 108695, 146957, 42825] avg=131072.0000 max=208605 max_rank=1 min=42825 min_rank=7 imbalance_ratio=1.591530 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[132381, 132384, 132182, 125378, 132384, 132383, 129102, 132382] avg=131072.0000 max=132384 max_rank=1 min=125378 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107742 max_load_reduction_pct=0.448689 moved_to_replicas=299136 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] avg=131072.0000 max=132384 max_rank=6 min=127034 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=76221 max_load_reduction_pct=0.365384 moved_to_replicas=256296 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] avg=131072.0000 max=132384 max_rank=6 min=127034 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=76221 max_load_reduction_pct=0.365384 moved_to_replicas=256296 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[123715, 208605, 135911, 113132, 168736, 108695, 146957, 42825] avg=131072.0000 max=208605 max_rank=1 min=42825 min_rank=7 imbalance_ratio=1.591530 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[91208, 91615, 220386, 41072, 159447, 139334, 171286, 134228] avg=131072.0000 max=220386 max_rank=2 min=41072 min_rank=3 imbalance_ratio=1.681412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[123715, 208605, 135911, 113132, 168736, 108695, 146957, 42825] avg=131072.0000 max=208605 max_rank=1 min=42825 min_rank=7 imbalance_ratio=1.591530 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[123715, 208605, 135911, 113132, 168736, 108695, 146957, 42825] avg=131072.0000 max=208605 max_rank=1 min=42825 min_rank=7 imbalance_ratio=1.591530 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] avg=131072.0000 max=132384 max_rank=6 min=127034 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=76221 max_load_reduction_pct=0.365384 moved_to_replicas=256296 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[91208, 91615, 220386, 41072, 159447, 139334, 171286, 134228] avg=131072.0000 max=220386 max_rank=2 min=41072 min_rank=3 imbalance_ratio=1.681412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[123715, 208605, 135911, 113132, 168736, 108695, 146957, 42825] avg=131072.0000 max=208605 max_rank=1 min=42825 min_rank=7 imbalance_ratio=1.591530 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] avg=131072.0000 max=132384 max_rank=6 min=127034 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=76221 max_load_reduction_pct=0.365384 moved_to_replicas=256296 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] avg=131072.0000 max=132386 max_rank=3 min=126086 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=88000 max_load_reduction_pct=0.399299 moved_to_replicas=311244 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[91208, 91615, 220386, 41072, 159447, 139334, 171286, 134228] avg=131072.0000 max=220386 max_rank=2 min=41072 min_rank=3 imbalance_ratio=1.681412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] avg=131072.0000 max=132384 max_rank=6 min=127034 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=76221 max_load_reduction_pct=0.365384 moved_to_replicas=256296 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] avg=131072.0000 max=132384 max_rank=6 min=127034 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=76221 max_load_reduction_pct=0.365384 moved_to_replicas=256296 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] avg=131072.0000 max=132386 max_rank=3 min=126086 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=88000 max_load_reduction_pct=0.399299 moved_to_replicas=311244 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[127243, 132383, 132383, 132383, 132383, 132383, 132384, 127034] avg=131072.0000 max=132384 max_rank=6 min=127034 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=76221 max_load_reduction_pct=0.365384 moved_to_replicas=256296 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[91208, 91615, 220386, 41072, 159447, 139334, 171286, 134228] avg=131072.0000 max=220386 max_rank=2 min=41072 min_rank=3 imbalance_ratio=1.681412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] avg=131072.0000 max=132386 max_rank=3 min=126086 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=88000 max_load_reduction_pct=0.399299 moved_to_replicas=311244 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[91208, 91615, 220386, 41072, 159447, 139334, 171286, 134228] avg=131072.0000 max=220386 max_rank=2 min=41072 min_rank=3 imbalance_ratio=1.681412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[149668, 186200, 133113, 185904, 149870, 1782, 85522, 156517] avg=131072.0000 max=186200 max_rank=1 min=1782 min_rank=5 imbalance_ratio=1.420593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[91208, 91615, 220386, 41072, 159447, 139334, 171286, 134228] avg=131072.0000 max=220386 max_rank=2 min=41072 min_rank=3 imbalance_ratio=1.681412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[91208, 91615, 220386, 41072, 159447, 139334, 171286, 134228] avg=131072.0000 max=220386 max_rank=2 min=41072 min_rank=3 imbalance_ratio=1.681412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] avg=131072.0000 max=132386 max_rank=3 min=126086 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=88000 max_load_reduction_pct=0.399299 moved_to_replicas=311244 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[149668, 186200, 133113, 185904, 149870, 1782, 85522, 156517] avg=131072.0000 max=186200 max_rank=1 min=1782 min_rank=5 imbalance_ratio=1.420593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[91208, 91615, 220386, 41072, 159447, 139334, 171286, 134228] avg=131072.0000 max=220386 max_rank=2 min=41072 min_rank=3 imbalance_ratio=1.681412 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] avg=131072.0000 max=132386 max_rank=3 min=126086 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=88000 max_load_reduction_pct=0.399299 moved_to_replicas=311244 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] avg=131072.0000 max=132384 max_rank=4 min=124543 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53816 max_load_reduction_pct=0.289023 moved_to_replicas=320950 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[149668, 186200, 133113, 185904, 149870, 1782, 85522, 156517] avg=131072.0000 max=186200 max_rank=1 min=1782 min_rank=5 imbalance_ratio=1.420593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] avg=131072.0000 max=132386 max_rank=3 min=126086 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=88000 max_load_reduction_pct=0.399299 moved_to_replicas=311244 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] avg=131072.0000 max=132386 max_rank=3 min=126086 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=88000 max_load_reduction_pct=0.399299 moved_to_replicas=311244 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] avg=131072.0000 max=132384 max_rank=4 min=124543 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53816 max_load_reduction_pct=0.289023 moved_to_replicas=320950 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[128193, 126086, 132383, 132386, 132382, 132382, 132381, 132383] avg=131072.0000 max=132386 max_rank=3 min=126086 min_rank=1 imbalance_ratio=1.010025 max_load_reduction=88000 max_load_reduction_pct=0.399299 moved_to_replicas=311244 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] avg=131072.0000 max=132384 max_rank=4 min=124543 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53816 max_load_reduction_pct=0.289023 moved_to_replicas=320950 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[149668, 186200, 133113, 185904, 149870, 1782, 85522, 156517] avg=131072.0000 max=186200 max_rank=1 min=1782 min_rank=5 imbalance_ratio=1.420593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[149668, 186200, 133113, 185904, 149870, 1782, 85522, 156517] avg=131072.0000 max=186200 max_rank=1 min=1782 min_rank=5 imbalance_ratio=1.420593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[145448, 134902, 46292, 130054, 139044, 162307, 169447, 121082] avg=131072.0000 max=169447 max_rank=6 min=46292 min_rank=2 imbalance_ratio=1.292778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[149668, 186200, 133113, 185904, 149870, 1782, 85522, 156517] avg=131072.0000 max=186200 max_rank=1 min=1782 min_rank=5 imbalance_ratio=1.420593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[145448, 134902, 46292, 130054, 139044, 162307, 169447, 121082] avg=131072.0000 max=169447 max_rank=6 min=46292 min_rank=2 imbalance_ratio=1.292778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[149668, 186200, 133113, 185904, 149870, 1782, 85522, 156517] avg=131072.0000 max=186200 max_rank=1 min=1782 min_rank=5 imbalance_ratio=1.420593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] avg=131072.0000 max=132384 max_rank=4 min=124543 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53816 max_load_reduction_pct=0.289023 moved_to_replicas=320950 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] avg=131072.0000 max=132384 max_rank=4 min=124543 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53816 max_load_reduction_pct=0.289023 moved_to_replicas=320950 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] avg=131072.0000 max=132384 max_rank=0 min=127744 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37063 max_load_reduction_pct=0.218729 moved_to_replicas=195024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[149668, 186200, 133113, 185904, 149870, 1782, 85522, 156517] avg=131072.0000 max=186200 max_rank=1 min=1782 min_rank=5 imbalance_ratio=1.420593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[145448, 134902, 46292, 130054, 139044, 162307, 169447, 121082] avg=131072.0000 max=169447 max_rank=6 min=46292 min_rank=2 imbalance_ratio=1.292778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] avg=131072.0000 max=132384 max_rank=4 min=124543 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53816 max_load_reduction_pct=0.289023 moved_to_replicas=320950 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] avg=131072.0000 max=132384 max_rank=0 min=127744 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37063 max_load_reduction_pct=0.218729 moved_to_replicas=195024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] avg=131072.0000 max=132384 max_rank=4 min=124543 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53816 max_load_reduction_pct=0.289023 moved_to_replicas=320950 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 132384, 129736, 124543, 132383] avg=131072.0000 max=132384 max_rank=4 min=124543 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53816 max_load_reduction_pct=0.289023 moved_to_replicas=320950 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] avg=131072.0000 max=132384 max_rank=0 min=127744 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37063 max_load_reduction_pct=0.218729 moved_to_replicas=195024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[145448, 134902, 46292, 130054, 139044, 162307, 169447, 121082] avg=131072.0000 max=169447 max_rank=6 min=46292 min_rank=2 imbalance_ratio=1.292778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[145448, 134902, 46292, 130054, 139044, 162307, 169447, 121082] avg=131072.0000 max=169447 max_rank=6 min=46292 min_rank=2 imbalance_ratio=1.292778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[102353, 130542, 195021, 97673, 177095, 71875, 96521, 177496] avg=131072.0000 max=195021 max_rank=2 min=71875 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[145448, 134902, 46292, 130054, 139044, 162307, 169447, 121082] avg=131072.0000 max=169447 max_rank=6 min=46292 min_rank=2 imbalance_ratio=1.292778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] avg=131072.0000 max=132384 max_rank=0 min=127744 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37063 max_load_reduction_pct=0.218729 moved_to_replicas=195024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] avg=131072.0000 max=132384 max_rank=0 min=127744 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37063 max_load_reduction_pct=0.218729 moved_to_replicas=195024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] avg=131072.0000 max=132385 max_rank=7 min=126464 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=62636 max_load_reduction_pct=0.321176 moved_to_replicas=304007 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[102353, 130542, 195021, 97673, 177095, 71875, 96521, 177496] avg=131072.0000 max=195021 max_rank=2 min=71875 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[145448, 134902, 46292, 130054, 139044, 162307, 169447, 121082] avg=131072.0000 max=169447 max_rank=6 min=46292 min_rank=2 imbalance_ratio=1.292778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[102353, 130542, 195021, 97673, 177095, 71875, 96521, 177496] avg=131072.0000 max=195021 max_rank=2 min=71875 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[145448, 134902, 46292, 130054, 139044, 162307, 169447, 121082] avg=131072.0000 max=169447 max_rank=6 min=46292 min_rank=2 imbalance_ratio=1.292778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] avg=131072.0000 max=132384 max_rank=0 min=127744 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37063 max_load_reduction_pct=0.218729 moved_to_replicas=195024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] avg=131072.0000 max=132385 max_rank=7 min=126464 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=62636 max_load_reduction_pct=0.321176 moved_to_replicas=304007 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] avg=131072.0000 max=132384 max_rank=0 min=127744 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37063 max_load_reduction_pct=0.218729 moved_to_replicas=195024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] avg=131072.0000 max=132385 max_rank=7 min=126464 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=62636 max_load_reduction_pct=0.321176 moved_to_replicas=304007 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[132384, 132383, 128863, 130054, 132382, 132383, 132383, 127744] avg=131072.0000 max=132384 max_rank=0 min=127744 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=37063 max_load_reduction_pct=0.218729 moved_to_replicas=195024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[102353, 130542, 195021, 97673, 177095, 71875, 96521, 177496] avg=131072.0000 max=195021 max_rank=2 min=71875 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[102353, 130542, 195021, 97673, 177095, 71875, 96521, 177496] avg=131072.0000 max=195021 max_rank=2 min=71875 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[131085, 157605, 105376, 188807, 110525, 113360, 161254, 80564] avg=131072.0000 max=188807 max_rank=3 min=80564 min_rank=7 imbalance_ratio=1.440483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[102353, 130542, 195021, 97673, 177095, 71875, 96521, 177496] avg=131072.0000 max=195021 max_rank=2 min=71875 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] avg=131072.0000 max=132385 max_rank=7 min=126464 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=62636 max_load_reduction_pct=0.321176 moved_to_replicas=304007 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] avg=131072.0000 max=132385 max_rank=7 min=126464 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=62636 max_load_reduction_pct=0.321176 moved_to_replicas=304007 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] avg=131072.0000 max=132384 max_rank=3 min=126450 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56423 max_load_reduction_pct=0.298840 moved_to_replicas=219183 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[131085, 157605, 105376, 188807, 110525, 113360, 161254, 80564] avg=131072.0000 max=188807 max_rank=3 min=80564 min_rank=7 imbalance_ratio=1.440483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[102353, 130542, 195021, 97673, 177095, 71875, 96521, 177496] avg=131072.0000 max=195021 max_rank=2 min=71875 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[131085, 157605, 105376, 188807, 110525, 113360, 161254, 80564] avg=131072.0000 max=188807 max_rank=3 min=80564 min_rank=7 imbalance_ratio=1.440483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[102353, 130542, 195021, 97673, 177095, 71875, 96521, 177496] avg=131072.0000 max=195021 max_rank=2 min=71875 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] avg=131072.0000 max=132385 max_rank=7 min=126464 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=62636 max_load_reduction_pct=0.321176 moved_to_replicas=304007 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] avg=131072.0000 max=132384 max_rank=3 min=126450 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56423 max_load_reduction_pct=0.298840 moved_to_replicas=219183 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] avg=131072.0000 max=132384 max_rank=3 min=126450 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56423 max_load_reduction_pct=0.298840 moved_to_replicas=219183 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] avg=131072.0000 max=132385 max_rank=7 min=126464 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=62636 max_load_reduction_pct=0.321176 moved_to_replicas=304007 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[131627, 130542, 132383, 130411, 132382, 132382, 126464, 132385] avg=131072.0000 max=132385 max_rank=7 min=126464 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=62636 max_load_reduction_pct=0.321176 moved_to_replicas=304007 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[131085, 157605, 105376, 188807, 110525, 113360, 161254, 80564] avg=131072.0000 max=188807 max_rank=3 min=80564 min_rank=7 imbalance_ratio=1.440483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[131085, 157605, 105376, 188807, 110525, 113360, 161254, 80564] avg=131072.0000 max=188807 max_rank=3 min=80564 min_rank=7 imbalance_ratio=1.440483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[88698, 132203, 110928, 276905, 76439, 136682, 138230, 88491] avg=131072.0000 max=276905 max_rank=3 min=76439 min_rank=4 imbalance_ratio=2.112617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[131085, 157605, 105376, 188807, 110525, 113360, 161254, 80564] avg=131072.0000 max=188807 max_rank=3 min=80564 min_rank=7 imbalance_ratio=1.440483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] avg=131072.0000 max=132384 max_rank=3 min=126450 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56423 max_load_reduction_pct=0.298840 moved_to_replicas=219183 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[88698, 132203, 110928, 276905, 76439, 136682, 138230, 88491] avg=131072.0000 max=276905 max_rank=3 min=76439 min_rank=4 imbalance_ratio=2.112617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] avg=131072.0000 max=132384 max_rank=3 min=126450 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56423 max_load_reduction_pct=0.298840 moved_to_replicas=219183 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] avg=131072.0000 max=132384 max_rank=4 min=127159 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=144521 max_load_reduction_pct=0.521915 moved_to_replicas=292891 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] avg=131072.0000 max=132384 max_rank=3 min=126450 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56423 max_load_reduction_pct=0.298840 moved_to_replicas=219183 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[88698, 132203, 110928, 276905, 76439, 136682, 138230, 88491] avg=131072.0000 max=276905 max_rank=3 min=76439 min_rank=4 imbalance_ratio=2.112617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[131085, 157605, 105376, 188807, 110525, 113360, 161254, 80564] avg=131072.0000 max=188807 max_rank=3 min=80564 min_rank=7 imbalance_ratio=1.440483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[131085, 157605, 105376, 188807, 110525, 113360, 161254, 80564] avg=131072.0000 max=188807 max_rank=3 min=80564 min_rank=7 imbalance_ratio=1.440483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] avg=131072.0000 max=132384 max_rank=4 min=127159 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=144521 max_load_reduction_pct=0.521915 moved_to_replicas=292891 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] avg=131072.0000 max=132384 max_rank=4 min=127159 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=144521 max_load_reduction_pct=0.521915 moved_to_replicas=292891 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] avg=131072.0000 max=132384 max_rank=3 min=126450 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56423 max_load_reduction_pct=0.298840 moved_to_replicas=219183 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[88698, 132203, 110928, 276905, 76439, 136682, 138230, 88491] avg=131072.0000 max=276905 max_rank=3 min=76439 min_rank=4 imbalance_ratio=2.112617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[131085, 132383, 129126, 132384, 132383, 132382, 132383, 126450] avg=131072.0000 max=132384 max_rank=3 min=126450 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56423 max_load_reduction_pct=0.298840 moved_to_replicas=219183 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[88698, 132203, 110928, 276905, 76439, 136682, 138230, 88491] avg=131072.0000 max=276905 max_rank=3 min=76439 min_rank=4 imbalance_ratio=2.112617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[121342, 114899, 191219, 131339, 163524, 127641, 196341, 2271] avg=131072.0000 max=196341 max_rank=6 min=2271 min_rank=7 imbalance_ratio=1.497963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[88698, 132203, 110928, 276905, 76439, 136682, 138230, 88491] avg=131072.0000 max=276905 max_rank=3 min=76439 min_rank=4 imbalance_ratio=2.112617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[121342, 114899, 191219, 131339, 163524, 127641, 196341, 2271] avg=131072.0000 max=196341 max_rank=6 min=2271 min_rank=7 imbalance_ratio=1.497963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] avg=131072.0000 max=132384 max_rank=4 min=127159 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=144521 max_load_reduction_pct=0.521915 moved_to_replicas=292891 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] avg=131072.0000 max=132384 max_rank=4 min=127159 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=144521 max_load_reduction_pct=0.521915 moved_to_replicas=292891 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] avg=131072.0000 max=132383 max_rank=0 min=127641 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=63958 max_load_reduction_pct=0.325750 moved_to_replicas=310175 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] avg=131072.0000 max=132384 max_rank=4 min=127159 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=144521 max_load_reduction_pct=0.521915 moved_to_replicas=292891 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[121342, 114899, 191219, 131339, 163524, 127641, 196341, 2271] avg=131072.0000 max=196341 max_rank=6 min=2271 min_rank=7 imbalance_ratio=1.497963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[88698, 132203, 110928, 276905, 76439, 136682, 138230, 88491] avg=131072.0000 max=276905 max_rank=3 min=76439 min_rank=4 imbalance_ratio=2.112617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[88698, 132203, 110928, 276905, 76439, 136682, 138230, 88491] avg=131072.0000 max=276905 max_rank=3 min=76439 min_rank=4 imbalance_ratio=2.112617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] avg=131072.0000 max=132383 max_rank=0 min=127641 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=63958 max_load_reduction_pct=0.325750 moved_to_replicas=310175 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[121342, 114899, 191219, 131339, 163524, 127641, 196341, 2271] avg=131072.0000 max=196341 max_rank=6 min=2271 min_rank=7 imbalance_ratio=1.497963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] avg=131072.0000 max=132383 max_rank=0 min=127641 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=63958 max_load_reduction_pct=0.325750 moved_to_replicas=310175 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] avg=131072.0000 max=132384 max_rank=4 min=127159 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=144521 max_load_reduction_pct=0.521915 moved_to_replicas=292891 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[127159, 132203, 132383, 132382, 132384, 132382, 132382, 127301] avg=131072.0000 max=132384 max_rank=4 min=127159 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=144521 max_load_reduction_pct=0.521915 moved_to_replicas=292891 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[121342, 114899, 191219, 131339, 163524, 127641, 196341, 2271] avg=131072.0000 max=196341 max_rank=6 min=2271 min_rank=7 imbalance_ratio=1.497963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[166701, 225627, 105856, 97039, 42869, 117363, 131709, 161412] avg=131072.0000 max=225627 max_rank=1 min=42869 min_rank=4 imbalance_ratio=1.721397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[121342, 114899, 191219, 131339, 163524, 127641, 196341, 2271] avg=131072.0000 max=196341 max_rank=6 min=2271 min_rank=7 imbalance_ratio=1.497963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[166701, 225627, 105856, 97039, 42869, 117363, 131709, 161412] avg=131072.0000 max=225627 max_rank=1 min=42869 min_rank=4 imbalance_ratio=1.721397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] avg=131072.0000 max=132383 max_rank=0 min=127641 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=63958 max_load_reduction_pct=0.325750 moved_to_replicas=310175 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] avg=131072.0000 max=132383 max_rank=0 min=127641 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=63958 max_load_reduction_pct=0.325750 moved_to_replicas=310175 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] avg=131072.0000 max=132384 max_rank=1 min=124830 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=93243 max_load_reduction_pct=0.413262 moved_to_replicas=314240 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[166701, 225627, 105856, 97039, 42869, 117363, 131709, 161412] avg=131072.0000 max=225627 max_rank=1 min=42869 min_rank=4 imbalance_ratio=1.721397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] avg=131072.0000 max=132383 max_rank=0 min=127641 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=63958 max_load_reduction_pct=0.325750 moved_to_replicas=310175 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[121342, 114899, 191219, 131339, 163524, 127641, 196341, 2271] avg=131072.0000 max=196341 max_rank=6 min=2271 min_rank=7 imbalance_ratio=1.497963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[121342, 114899, 191219, 131339, 163524, 127641, 196341, 2271] avg=131072.0000 max=196341 max_rank=6 min=2271 min_rank=7 imbalance_ratio=1.497963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] avg=131072.0000 max=132384 max_rank=1 min=124830 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=93243 max_load_reduction_pct=0.413262 moved_to_replicas=314240 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] avg=131072.0000 max=132384 max_rank=1 min=124830 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=93243 max_load_reduction_pct=0.413262 moved_to_replicas=314240 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[166701, 225627, 105856, 97039, 42869, 117363, 131709, 161412] avg=131072.0000 max=225627 max_rank=1 min=42869 min_rank=4 imbalance_ratio=1.721397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] avg=131072.0000 max=132383 max_rank=0 min=127641 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=63958 max_load_reduction_pct=0.325750 moved_to_replicas=310175 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132382, 131339, 132383, 127641, 132383, 127682] avg=131072.0000 max=132383 max_rank=0 min=127641 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=63958 max_load_reduction_pct=0.325750 moved_to_replicas=310175 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[166701, 225627, 105856, 97039, 42869, 117363, 131709, 161412] avg=131072.0000 max=225627 max_rank=1 min=42869 min_rank=4 imbalance_ratio=1.721397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[134650, 141767, 130638, 129448, 130548, 129463, 84675, 167387] avg=131072.0000 max=167387 max_rank=7 min=84675 min_rank=6 imbalance_ratio=1.277061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[166701, 225627, 105856, 97039, 42869, 117363, 131709, 161412] avg=131072.0000 max=225627 max_rank=1 min=42869 min_rank=4 imbalance_ratio=1.721397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[134650, 141767, 130638, 129448, 130548, 129463, 84675, 167387] avg=131072.0000 max=167387 max_rank=7 min=84675 min_rank=6 imbalance_ratio=1.277061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] avg=131072.0000 max=132384 max_rank=1 min=124830 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=93243 max_load_reduction_pct=0.413262 moved_to_replicas=314240 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] avg=131072.0000 max=132384 max_rank=1 min=124830 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=93243 max_load_reduction_pct=0.413262 moved_to_replicas=314240 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] avg=131072.0000 max=132383 max_rank=0 min=129448 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35004 max_load_reduction_pct=0.209120 moved_to_replicas=95893 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[134650, 141767, 130638, 129448, 130548, 129463, 84675, 167387] avg=131072.0000 max=167387 max_rank=7 min=84675 min_rank=6 imbalance_ratio=1.277061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] avg=131072.0000 max=132384 max_rank=1 min=124830 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=93243 max_load_reduction_pct=0.413262 moved_to_replicas=314240 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] avg=131072.0000 max=132383 max_rank=0 min=129448 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35004 max_load_reduction_pct=0.209120 moved_to_replicas=95893 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[166701, 225627, 105856, 97039, 42869, 117363, 131709, 161412] avg=131072.0000 max=225627 max_rank=1 min=42869 min_rank=4 imbalance_ratio=1.721397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[166701, 225627, 105856, 97039, 42869, 117363, 131709, 161412] avg=131072.0000 max=225627 max_rank=1 min=42869 min_rank=4 imbalance_ratio=1.721397 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[134650, 141767, 130638, 129448, 130548, 129463, 84675, 167387] avg=131072.0000 max=167387 max_rank=7 min=84675 min_rank=6 imbalance_ratio=1.277061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] avg=131072.0000 max=132383 max_rank=0 min=129448 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35004 max_load_reduction_pct=0.209120 moved_to_replicas=95893 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] avg=131072.0000 max=132384 max_rank=1 min=124830 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=93243 max_load_reduction_pct=0.413262 moved_to_replicas=314240 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132382, 124830, 132383, 130122, 131709, 132383] avg=131072.0000 max=132384 max_rank=1 min=124830 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=93243 max_load_reduction_pct=0.413262 moved_to_replicas=314240 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[134650, 141767, 130638, 129448, 130548, 129463, 84675, 167387] avg=131072.0000 max=167387 max_rank=7 min=84675 min_rank=6 imbalance_ratio=1.277061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[122379, 157973, 100135, 123995, 152604, 104411, 99207, 187872] avg=131072.0000 max=187872 max_rank=7 min=99207 min_rank=6 imbalance_ratio=1.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[134650, 141767, 130638, 129448, 130548, 129463, 84675, 167387] avg=131072.0000 max=167387 max_rank=7 min=84675 min_rank=6 imbalance_ratio=1.277061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[122379, 157973, 100135, 123995, 152604, 104411, 99207, 187872] avg=131072.0000 max=187872 max_rank=7 min=99207 min_rank=6 imbalance_ratio=1.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] avg=131072.0000 max=132383 max_rank=0 min=129448 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35004 max_load_reduction_pct=0.209120 moved_to_replicas=95893 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[122379, 157973, 100135, 123995, 152604, 104411, 99207, 187872] avg=131072.0000 max=187872 max_rank=7 min=99207 min_rank=6 imbalance_ratio=1.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] avg=131072.0000 max=132383 max_rank=0 min=129448 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35004 max_load_reduction_pct=0.209120 moved_to_replicas=95893 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] avg=131072.0000 max=132384 max_rank=0 min=124715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55488 max_load_reduction_pct=0.295350 moved_to_replicas=203251 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] avg=131072.0000 max=132383 max_rank=0 min=129448 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35004 max_load_reduction_pct=0.209120 moved_to_replicas=95893 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] avg=131072.0000 max=132384 max_rank=0 min=124715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55488 max_load_reduction_pct=0.295350 moved_to_replicas=203251 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[134650, 141767, 130638, 129448, 130548, 129463, 84675, 167387] avg=131072.0000 max=167387 max_rank=7 min=84675 min_rank=6 imbalance_ratio=1.277061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[134650, 141767, 130638, 129448, 130548, 129463, 84675, 167387] avg=131072.0000 max=167387 max_rank=7 min=84675 min_rank=6 imbalance_ratio=1.277061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[122379, 157973, 100135, 123995, 152604, 104411, 99207, 187872] avg=131072.0000 max=187872 max_rank=7 min=99207 min_rank=6 imbalance_ratio=1.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] avg=131072.0000 max=132384 max_rank=0 min=124715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55488 max_load_reduction_pct=0.295350 moved_to_replicas=203251 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] avg=131072.0000 max=132383 max_rank=0 min=129448 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35004 max_load_reduction_pct=0.209120 moved_to_replicas=95893 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132383, 130638, 129448, 130548, 129463, 131330, 132383] avg=131072.0000 max=132383 max_rank=0 min=129448 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35004 max_load_reduction_pct=0.209120 moved_to_replicas=95893 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[122379, 157973, 100135, 123995, 152604, 104411, 99207, 187872] avg=131072.0000 max=187872 max_rank=7 min=99207 min_rank=6 imbalance_ratio=1.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[164092, 141484, 124843, 127231, 165791, 137187, 98992, 88956] avg=131072.0000 max=165791 max_rank=4 min=88956 min_rank=7 imbalance_ratio=1.264885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[164092, 141484, 124843, 127231, 165791, 137187, 98992, 88956] avg=131072.0000 max=165791 max_rank=4 min=88956 min_rank=7 imbalance_ratio=1.264885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[122379, 157973, 100135, 123995, 152604, 104411, 99207, 187872] avg=131072.0000 max=187872 max_rank=7 min=99207 min_rank=6 imbalance_ratio=1.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] avg=131072.0000 max=132384 max_rank=0 min=124715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55488 max_load_reduction_pct=0.295350 moved_to_replicas=203251 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[164092, 141484, 124843, 127231, 165791, 137187, 98992, 88956] avg=131072.0000 max=165791 max_rank=4 min=88956 min_rank=7 imbalance_ratio=1.264885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] avg=131072.0000 max=132384 max_rank=5 min=127231 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33407 max_load_reduction_pct=0.201501 moved_to_replicas=148223 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] avg=131072.0000 max=132384 max_rank=0 min=124715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55488 max_load_reduction_pct=0.295350 moved_to_replicas=203251 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] avg=131072.0000 max=132384 max_rank=5 min=127231 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33407 max_load_reduction_pct=0.201501 moved_to_replicas=148223 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[122379, 157973, 100135, 123995, 152604, 104411, 99207, 187872] avg=131072.0000 max=187872 max_rank=7 min=99207 min_rank=6 imbalance_ratio=1.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[122379, 157973, 100135, 123995, 152604, 104411, 99207, 187872] avg=131072.0000 max=187872 max_rank=7 min=99207 min_rank=6 imbalance_ratio=1.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] avg=131072.0000 max=132384 max_rank=0 min=124715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55488 max_load_reduction_pct=0.295350 moved_to_replicas=203251 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] avg=131072.0000 max=132384 max_rank=5 min=127231 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33407 max_load_reduction_pct=0.201501 moved_to_replicas=148223 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[164092, 141484, 124843, 127231, 165791, 137187, 98992, 88956] avg=131072.0000 max=165791 max_rank=4 min=88956 min_rank=7 imbalance_ratio=1.264885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] avg=131072.0000 max=132384 max_rank=0 min=124715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55488 max_load_reduction_pct=0.295350 moved_to_replicas=203251 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[132384, 132382, 132382, 124715, 132382, 130002, 131945, 132384] avg=131072.0000 max=132384 max_rank=0 min=124715 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55488 max_load_reduction_pct=0.295350 moved_to_replicas=203251 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[149196, 63129, 176333, 228853, 166684, 83107, 114825, 66449] avg=131072.0000 max=228853 max_rank=3 min=63129 min_rank=1 imbalance_ratio=1.746010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[164092, 141484, 124843, 127231, 165791, 137187, 98992, 88956] avg=131072.0000 max=165791 max_rank=4 min=88956 min_rank=7 imbalance_ratio=1.264885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[149196, 63129, 176333, 228853, 166684, 83107, 114825, 66449] avg=131072.0000 max=228853 max_rank=3 min=63129 min_rank=1 imbalance_ratio=1.746010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[164092, 141484, 124843, 127231, 165791, 137187, 98992, 88956] avg=131072.0000 max=165791 max_rank=4 min=88956 min_rank=7 imbalance_ratio=1.264885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] avg=131072.0000 max=132384 max_rank=5 min=127231 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33407 max_load_reduction_pct=0.201501 moved_to_replicas=148223 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[149196, 63129, 176333, 228853, 166684, 83107, 114825, 66449] avg=131072.0000 max=228853 max_rank=3 min=63129 min_rank=1 imbalance_ratio=1.746010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125699 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96469 max_load_reduction_pct=0.421533 moved_to_replicas=370967 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] avg=131072.0000 max=132384 max_rank=5 min=127231 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33407 max_load_reduction_pct=0.201501 moved_to_replicas=148223 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125699 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96469 max_load_reduction_pct=0.421533 moved_to_replicas=370967 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] avg=131072.0000 max=132384 max_rank=5 min=127231 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33407 max_load_reduction_pct=0.201501 moved_to_replicas=148223 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[164092, 141484, 124843, 127231, 165791, 137187, 98992, 88956] avg=131072.0000 max=165791 max_rank=4 min=88956 min_rank=7 imbalance_ratio=1.264885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[164092, 141484, 124843, 127231, 165791, 137187, 98992, 88956] avg=131072.0000 max=165791 max_rank=4 min=88956 min_rank=7 imbalance_ratio=1.264885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125699 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96469 max_load_reduction_pct=0.421533 moved_to_replicas=370967 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[149196, 63129, 176333, 228853, 166684, 83107, 114825, 66449] avg=131072.0000 max=228853 max_rank=3 min=63129 min_rank=1 imbalance_ratio=1.746010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[143842, 52807, 139324, 151125, 174063, 98829, 65412, 223174] avg=131072.0000 max=223174 max_rank=7 min=52807 min_rank=1 imbalance_ratio=1.702682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[149196, 63129, 176333, 228853, 166684, 83107, 114825, 66449] avg=131072.0000 max=228853 max_rank=3 min=63129 min_rank=1 imbalance_ratio=1.746010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] avg=131072.0000 max=132384 max_rank=5 min=127231 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33407 max_load_reduction_pct=0.201501 moved_to_replicas=148223 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[143842, 52807, 139324, 151125, 174063, 98829, 65412, 223174] avg=131072.0000 max=223174 max_rank=7 min=52807 min_rank=1 imbalance_ratio=1.702682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132381, 129646, 127231, 132383, 132384, 131390, 130778] avg=131072.0000 max=132384 max_rank=5 min=127231 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=33407 max_load_reduction_pct=0.201501 moved_to_replicas=148223 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125699 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96469 max_load_reduction_pct=0.421533 moved_to_replicas=370967 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[149196, 63129, 176333, 228853, 166684, 83107, 114825, 66449] avg=131072.0000 max=228853 max_rank=3 min=63129 min_rank=1 imbalance_ratio=1.746010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[143842, 52807, 139324, 151125, 174063, 98829, 65412, 223174] avg=131072.0000 max=223174 max_rank=7 min=52807 min_rank=1 imbalance_ratio=1.702682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] avg=131072.0000 max=132386 max_rank=0 min=126546 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=90788 max_load_reduction_pct=0.406804 moved_to_replicas=343326 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125699 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96469 max_load_reduction_pct=0.421533 moved_to_replicas=370967 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] avg=131072.0000 max=132386 max_rank=0 min=126546 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=90788 max_load_reduction_pct=0.406804 moved_to_replicas=343326 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125699 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96469 max_load_reduction_pct=0.421533 moved_to_replicas=370967 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] avg=131072.0000 max=132386 max_rank=0 min=126546 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=90788 max_load_reduction_pct=0.406804 moved_to_replicas=343326 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[149196, 63129, 176333, 228853, 166684, 83107, 114825, 66449] avg=131072.0000 max=228853 max_rank=3 min=63129 min_rank=1 imbalance_ratio=1.746010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[149196, 63129, 176333, 228853, 166684, 83107, 114825, 66449] avg=131072.0000 max=228853 max_rank=3 min=63129 min_rank=1 imbalance_ratio=1.746010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[143842, 52807, 139324, 151125, 174063, 98829, 65412, 223174] avg=131072.0000 max=223174 max_rank=7 min=52807 min_rank=1 imbalance_ratio=1.702682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[143842, 52807, 139324, 151125, 174063, 98829, 65412, 223174] avg=131072.0000 max=223174 max_rank=7 min=52807 min_rank=1 imbalance_ratio=1.702682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125699 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96469 max_load_reduction_pct=0.421533 moved_to_replicas=370967 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[132383, 128578, 132383, 132383, 132383, 125699, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125699 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96469 max_load_reduction_pct=0.421533 moved_to_replicas=370967 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] avg=131072.0000 max=132386 max_rank=0 min=126546 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=90788 max_load_reduction_pct=0.406804 moved_to_replicas=343326 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[143842, 52807, 139324, 151125, 174063, 98829, 65412, 223174] avg=131072.0000 max=223174 max_rank=7 min=52807 min_rank=1 imbalance_ratio=1.702682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] avg=131072.0000 max=132386 max_rank=0 min=126546 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=90788 max_load_reduction_pct=0.406804 moved_to_replicas=343326 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] avg=131072.0000 max=132386 max_rank=0 min=126546 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=90788 max_load_reduction_pct=0.406804 moved_to_replicas=343326 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[143842, 52807, 139324, 151125, 174063, 98829, 65412, 223174] avg=131072.0000 max=223174 max_rank=7 min=52807 min_rank=1 imbalance_ratio=1.702682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[143842, 52807, 139324, 151125, 174063, 98829, 65412, 223174] avg=131072.0000 max=223174 max_rank=7 min=52807 min_rank=1 imbalance_ratio=1.702682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] avg=131072.0000 max=132386 max_rank=0 min=126546 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=90788 max_load_reduction_pct=0.406804 moved_to_replicas=343326 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[132386, 129275, 132382, 132384, 132383, 126546, 130837, 132383] avg=131072.0000 max=132386 max_rank=0 min=126546 min_rank=5 imbalance_ratio=1.010025 max_load_reduction=90788 max_load_reduction_pct=0.406804 moved_to_replicas=343326 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=4170758 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=382080 peak_demand=135018 required_capacity=1.030106 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=4210249 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344320 peak_demand=134403 required_capacity=1.025414 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=4177977 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374784 peak_demand=134402 required_capacity=1.025406 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=4191399 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=361216 peak_demand=135017 required_capacity=1.030098 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=4177958 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373504 peak_demand=135017 required_capacity=1.030098 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=4234968 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=318976 peak_demand=135019 required_capacity=1.030113 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=4201840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=352256 peak_demand=134404 required_capacity=1.025421 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=4189283 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=363264 peak_demand=132582 required_capacity=1.011520 + [2026-08-30 05:19:08] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 567.9 | throughput per GPU (TFLOP/s/GPU): 587.0 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.301636E+01 | loss scale: 1.0 | grad norm: 46.038 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=143616 expected_recv=253166 expected_overflow=0 exact_recv_per_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=143616 expected_recv=260281 expected_overflow=0 exact_recv_per_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=143616 expected_recv=259878 expected_overflow=0 exact_recv_per_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=143616 expected_recv=253166 actual_valid_recv=255232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=143616 expected_recv=260281 actual_valid_recv=262656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=255232 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=143616 expected_recv=259878 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=262656 slack_slots=24576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=143616 expected_recv=263147 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=143616 expected_recv=262172 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=143616 expected_recv=248004 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=143616 expected_recv=263147 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=143616 expected_recv=262172 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=143616 expected_recv=248004 actual_valid_recv=251008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267904 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=251008 slack_slots=36224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=143616 expected_recv=261999 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=143616 expected_recv=253381 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=143616 expected_recv=260467 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=143616 expected_recv=262241 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=143616 expected_recv=261999 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=143616 expected_recv=253381 actual_valid_recv=256128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=143616 expected_recv=260467 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=268032 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=265216 slack_slots=22016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=143616 expected_recv=262241 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=256128 slack_slots=31104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=265344 slack_slots=21888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=143616 expected_recv=261572 expected_overflow=0 exact_recv_per_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=143616 expected_recv=261288 expected_overflow=0 exact_recv_per_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=143616 expected_recv=253591 expected_overflow=0 exact_recv_per_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=143616 expected_recv=264770 expected_overflow=0 exact_recv_per_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=143616 expected_recv=261572 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=143616 expected_recv=261288 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=143616 expected_recv=261633 expected_overflow=0 exact_recv_per_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=143616 expected_recv=253591 actual_valid_recv=256384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=143616 expected_recv=264770 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=256384 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=143616 expected_recv=261633 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=143616 expected_recv=257408 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=143616 expected_recv=257408 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=143616 expected_recv=257629 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=143616 expected_recv=264762 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=143616 expected_recv=258286 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=259840 slack_slots=27392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=143616 expected_recv=257629 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=143616 expected_recv=264762 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=143616 expected_recv=258286 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=259968 slack_slots=27264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=260992 slack_slots=26240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=143616 expected_recv=264770 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=143616 expected_recv=257472 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=143616 expected_recv=259141 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=143616 expected_recv=264770 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=143616 expected_recv=257472 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=143616 expected_recv=259472 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=143616 expected_recv=262001 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=143616 expected_recv=259141 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=259712 slack_slots=27520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=143616 expected_recv=259472 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=143616 expected_recv=259707 expected_overflow=0 exact_recv_per_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=143616 expected_recv=262001 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=143616 expected_recv=259707 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=143616 expected_recv=261374 expected_overflow=0 exact_recv_per_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=143616 expected_recv=255873 expected_overflow=0 exact_recv_per_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=143616 expected_recv=261374 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=143616 expected_recv=255873 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=143616 expected_recv=261131 expected_overflow=0 exact_recv_per_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=258304 slack_slots=28928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=143616 expected_recv=261131 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=143616 expected_recv=254999 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=143616 expected_recv=264691 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=143616 expected_recv=254999 actual_valid_recv=257152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=143616 expected_recv=264691 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=257152 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=143616 expected_recv=259386 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=143616 expected_recv=261927 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=143616 expected_recv=261851 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=143616 expected_recv=259386 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=143616 expected_recv=261927 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=143616 expected_recv=261851 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=143616 expected_recv=262215 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=143616 expected_recv=262215 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=143616 expected_recv=258189 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=143616 expected_recv=259572 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=143616 expected_recv=258189 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=143616 expected_recv=260804 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=143616 expected_recv=259572 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=259968 slack_slots=27264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=143616 expected_recv=263522 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=143616 expected_recv=260804 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=143616 expected_recv=262459 expected_overflow=0 exact_recv_per_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=143616 expected_recv=263318 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=143616 expected_recv=263522 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=143616 expected_recv=262459 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=143616 expected_recv=263318 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=265856 slack_slots=21376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=143616 expected_recv=262758 expected_overflow=0 exact_recv_per_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=143616 expected_recv=262758 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=143616 expected_recv=256354 expected_overflow=0 exact_recv_per_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=143616 expected_recv=256515 expected_overflow=0 exact_recv_per_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=143616 expected_recv=256354 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=143616 expected_recv=256515 actual_valid_recv=258560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=258560 slack_slots=28672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=143616 expected_recv=261935 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=143616 expected_recv=261935 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=143616 expected_recv=263755 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=143616 expected_recv=248323 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=143616 expected_recv=263755 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=143616 expected_recv=248323 actual_valid_recv=250624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=143616 expected_recv=264074 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=265856 slack_slots=21376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=250624 slack_slots=36608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=143616 expected_recv=259348 expected_overflow=0 exact_recv_per_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=143616 expected_recv=259169 expected_overflow=0 exact_recv_per_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=143616 expected_recv=264074 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=143616 expected_recv=259348 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=143616 expected_recv=259169 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=143616 expected_recv=260868 expected_overflow=0 exact_recv_per_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=143616 expected_recv=262808 expected_overflow=0 exact_recv_per_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=143616 expected_recv=259884 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=143616 expected_recv=260868 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=143616 expected_recv=262808 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=143616 expected_recv=260661 expected_overflow=0 exact_recv_per_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=143616 expected_recv=259884 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=264832 slack_slots=22400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=143616 expected_recv=260661 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=143616 expected_recv=261359 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=143616 expected_recv=261516 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=143616 expected_recv=261359 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=143616 expected_recv=261516 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=143616 expected_recv=262194 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=143616 expected_recv=258383 expected_overflow=0 exact_recv_per_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=143616 expected_recv=262194 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=143616 expected_recv=258383 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=143616 expected_recv=257901 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=143616 expected_recv=259055 expected_overflow=0 exact_recv_per_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=143616 expected_recv=257901 actual_valid_recv=260224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=143616 expected_recv=259055 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=260224 slack_slots=27008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=261120 slack_slots=26112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=143616 expected_recv=260971 expected_overflow=0 exact_recv_per_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=143616 expected_recv=260332 expected_overflow=0 exact_recv_per_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=143616 expected_recv=260971 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=143616 expected_recv=260332 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=143616 expected_recv=264066 expected_overflow=0 exact_recv_per_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=143616 expected_recv=264066 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=143616 expected_recv=260379 expected_overflow=0 exact_recv_per_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=143616 expected_recv=262702 expected_overflow=0 exact_recv_per_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=143616 expected_recv=260379 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=143616 expected_recv=262702 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=143616 expected_recv=255977 expected_overflow=0 exact_recv_per_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=143616 expected_recv=262072 expected_overflow=0 exact_recv_per_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=143616 expected_recv=255977 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=143616 expected_recv=262072 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=257920 slack_slots=29312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=143616 expected_recv=255468 expected_overflow=0 exact_recv_per_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=143616 expected_recv=259077 expected_overflow=0 exact_recv_per_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=143616 expected_recv=255468 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=143616 expected_recv=259077 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=258304 slack_slots=28928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=143616 expected_recv=259272 expected_overflow=0 exact_recv_per_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=143616 expected_recv=261571 expected_overflow=0 exact_recv_per_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[88352, 123982, 142109, 124063, 146486, 157502, 169999, 96083] avg=131072.0000 max=169999 max_rank=6 min=88352 min_rank=0 imbalance_ratio=1.296989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=143616 expected_recv=259272 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=143616 expected_recv=261571 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=260992 slack_slots=26240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] avg=131072.0000 max=132384 max_rank=3 min=125828 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=37615 max_load_reduction_pct=0.221266 moved_to_replicas=169939 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[88352, 123982, 142109, 124063, 146486, 157502, 169999, 96083] avg=131072.0000 max=169999 max_rank=6 min=88352 min_rank=0 imbalance_ratio=1.296989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=143616 expected_recv=264472 expected_overflow=0 exact_recv_per_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[88352, 123982, 142109, 124063, 146486, 157502, 169999, 96083] avg=131072.0000 max=169999 max_rank=6 min=88352 min_rank=0 imbalance_ratio=1.296989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=143616 expected_recv=264472 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[88352, 123982, 142109, 124063, 146486, 157502, 169999, 96083] avg=131072.0000 max=169999 max_rank=6 min=88352 min_rank=0 imbalance_ratio=1.296989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[88352, 123982, 142109, 124063, 146486, 157502, 169999, 96083] avg=131072.0000 max=169999 max_rank=6 min=88352 min_rank=0 imbalance_ratio=1.296989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] avg=131072.0000 max=132384 max_rank=3 min=125828 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=37615 max_load_reduction_pct=0.221266 moved_to_replicas=169939 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[88352, 123982, 142109, 124063, 146486, 157502, 169999, 96083] avg=131072.0000 max=169999 max_rank=6 min=88352 min_rank=0 imbalance_ratio=1.296989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] avg=131072.0000 max=132384 max_rank=3 min=125828 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=37615 max_load_reduction_pct=0.221266 moved_to_replicas=169939 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[149416, 72279, 67821, 133002, 168876, 164572, 98387, 194223] avg=131072.0000 max=194223 max_rank=7 min=67821 min_rank=2 imbalance_ratio=1.481804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] avg=131072.0000 max=132384 max_rank=3 min=125828 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=37615 max_load_reduction_pct=0.221266 moved_to_replicas=169939 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[88352, 123982, 142109, 124063, 146486, 157502, 169999, 96083] avg=131072.0000 max=169999 max_rank=6 min=88352 min_rank=0 imbalance_ratio=1.296989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] avg=131072.0000 max=132384 max_rank=3 min=125828 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=37615 max_load_reduction_pct=0.221266 moved_to_replicas=169939 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] avg=131072.0000 max=132383 max_rank=1 min=123238 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=61840 max_load_reduction_pct=0.318397 moved_to_replicas=297697 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] avg=131072.0000 max=132384 max_rank=3 min=125828 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=37615 max_load_reduction_pct=0.221266 moved_to_replicas=169939 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[149416, 72279, 67821, 133002, 168876, 164572, 98387, 194223] avg=131072.0000 max=194223 max_rank=7 min=67821 min_rank=2 imbalance_ratio=1.481804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[88352, 123982, 142109, 124063, 146486, 157502, 169999, 96083] avg=131072.0000 max=169999 max_rank=6 min=88352 min_rank=0 imbalance_ratio=1.296989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[149416, 72279, 67821, 133002, 168876, 164572, 98387, 194223] avg=131072.0000 max=194223 max_rank=7 min=67821 min_rank=2 imbalance_ratio=1.481804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] avg=131072.0000 max=132384 max_rank=3 min=125828 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=37615 max_load_reduction_pct=0.221266 moved_to_replicas=169939 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[149416, 72279, 67821, 133002, 168876, 164572, 98387, 194223] avg=131072.0000 max=194223 max_rank=7 min=67821 min_rank=2 imbalance_ratio=1.481804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] avg=131072.0000 max=132383 max_rank=1 min=123238 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=61840 max_load_reduction_pct=0.318397 moved_to_replicas=297697 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[149416, 72279, 67821, 133002, 168876, 164572, 98387, 194223] avg=131072.0000 max=194223 max_rank=7 min=67821 min_rank=2 imbalance_ratio=1.481804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[165780, 206720, 98855, 109808, 98390, 125278, 168732, 75013] avg=131072.0000 max=206720 max_rank=1 min=75013 min_rank=7 imbalance_ratio=1.577148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] avg=131072.0000 max=132383 max_rank=1 min=123238 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=61840 max_load_reduction_pct=0.318397 moved_to_replicas=297697 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125828, 132382, 132384, 132383, 132382, 132383, 128451] avg=131072.0000 max=132384 max_rank=3 min=125828 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=37615 max_load_reduction_pct=0.221266 moved_to_replicas=169939 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[149416, 72279, 67821, 133002, 168876, 164572, 98387, 194223] avg=131072.0000 max=194223 max_rank=7 min=67821 min_rank=2 imbalance_ratio=1.481804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] avg=131072.0000 max=132383 max_rank=1 min=123238 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=61840 max_load_reduction_pct=0.318397 moved_to_replicas=297697 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[149416, 72279, 67821, 133002, 168876, 164572, 98387, 194223] avg=131072.0000 max=194223 max_rank=7 min=67821 min_rank=2 imbalance_ratio=1.481804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] avg=131072.0000 max=132385 max_rank=2 min=125910 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=74335 max_load_reduction_pct=0.359593 moved_to_replicas=290316 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] avg=131072.0000 max=132383 max_rank=1 min=123238 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=61840 max_load_reduction_pct=0.318397 moved_to_replicas=297697 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[165780, 206720, 98855, 109808, 98390, 125278, 168732, 75013] avg=131072.0000 max=206720 max_rank=1 min=75013 min_rank=7 imbalance_ratio=1.577148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] avg=131072.0000 max=132383 max_rank=1 min=123238 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=61840 max_load_reduction_pct=0.318397 moved_to_replicas=297697 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[165780, 206720, 98855, 109808, 98390, 125278, 168732, 75013] avg=131072.0000 max=206720 max_rank=1 min=75013 min_rank=7 imbalance_ratio=1.577148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] avg=131072.0000 max=132383 max_rank=1 min=123238 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=61840 max_load_reduction_pct=0.318397 moved_to_replicas=297697 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[149416, 72279, 67821, 133002, 168876, 164572, 98387, 194223] avg=131072.0000 max=194223 max_rank=7 min=67821 min_rank=2 imbalance_ratio=1.481804 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[165780, 206720, 98855, 109808, 98390, 125278, 168732, 75013] avg=131072.0000 max=206720 max_rank=1 min=75013 min_rank=7 imbalance_ratio=1.577148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] avg=131072.0000 max=132385 max_rank=2 min=125910 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=74335 max_load_reduction_pct=0.359593 moved_to_replicas=290316 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[78240, 130772, 146679, 66069, 141701, 165251, 192782, 127082] avg=131072.0000 max=192782 max_rank=6 min=66069 min_rank=3 imbalance_ratio=1.470810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[165780, 206720, 98855, 109808, 98390, 125278, 168732, 75013] avg=131072.0000 max=206720 max_rank=1 min=75013 min_rank=7 imbalance_ratio=1.577148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] avg=131072.0000 max=132385 max_rank=2 min=125910 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=74335 max_load_reduction_pct=0.359593 moved_to_replicas=290316 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123238, 132383, 132383, 132383, 131041, 132383] avg=131072.0000 max=132383 max_rank=1 min=123238 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=61840 max_load_reduction_pct=0.318397 moved_to_replicas=297697 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[165780, 206720, 98855, 109808, 98390, 125278, 168732, 75013] avg=131072.0000 max=206720 max_rank=1 min=75013 min_rank=7 imbalance_ratio=1.577148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] avg=131072.0000 max=132385 max_rank=2 min=125910 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=74335 max_load_reduction_pct=0.359593 moved_to_replicas=290316 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[165780, 206720, 98855, 109808, 98390, 125278, 168732, 75013] avg=131072.0000 max=206720 max_rank=1 min=75013 min_rank=7 imbalance_ratio=1.577148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] avg=131072.0000 max=132385 max_rank=6 min=127082 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60397 max_load_reduction_pct=0.313292 moved_to_replicas=232902 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] avg=131072.0000 max=132385 max_rank=2 min=125910 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=74335 max_load_reduction_pct=0.359593 moved_to_replicas=290316 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[78240, 130772, 146679, 66069, 141701, 165251, 192782, 127082] avg=131072.0000 max=192782 max_rank=6 min=66069 min_rank=3 imbalance_ratio=1.470810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] avg=131072.0000 max=132385 max_rank=2 min=125910 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=74335 max_load_reduction_pct=0.359593 moved_to_replicas=290316 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[78240, 130772, 146679, 66069, 141701, 165251, 192782, 127082] avg=131072.0000 max=192782 max_rank=6 min=66069 min_rank=3 imbalance_ratio=1.470810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] avg=131072.0000 max=132385 max_rank=2 min=125910 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=74335 max_load_reduction_pct=0.359593 moved_to_replicas=290316 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[165780, 206720, 98855, 109808, 98390, 125278, 168732, 75013] avg=131072.0000 max=206720 max_rank=1 min=75013 min_rank=7 imbalance_ratio=1.577148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[78240, 130772, 146679, 66069, 141701, 165251, 192782, 127082] avg=131072.0000 max=192782 max_rank=6 min=66069 min_rank=3 imbalance_ratio=1.470810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] avg=131072.0000 max=132385 max_rank=6 min=127082 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60397 max_load_reduction_pct=0.313292 moved_to_replicas=232902 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[163269, 181444, 98305, 119674, 162580, 97957, 89949, 135398] avg=131072.0000 max=181444 max_rank=1 min=89949 min_rank=6 imbalance_ratio=1.384308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[78240, 130772, 146679, 66069, 141701, 165251, 192782, 127082] avg=131072.0000 max=192782 max_rank=6 min=66069 min_rank=3 imbalance_ratio=1.470810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] avg=131072.0000 max=132385 max_rank=6 min=127082 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60397 max_load_reduction_pct=0.313292 moved_to_replicas=232902 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[78240, 130772, 146679, 66069, 141701, 165251, 192782, 127082] avg=131072.0000 max=192782 max_rank=6 min=66069 min_rank=3 imbalance_ratio=1.470810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 132385, 132382, 131134, 125910, 132383, 129616] avg=131072.0000 max=132385 max_rank=2 min=125910 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=74335 max_load_reduction_pct=0.359593 moved_to_replicas=290316 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] avg=131072.0000 max=132385 max_rank=1 min=125730 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=49059 max_load_reduction_pct=0.270381 moved_to_replicas=228723 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] avg=131072.0000 max=132385 max_rank=6 min=127082 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60397 max_load_reduction_pct=0.313292 moved_to_replicas=232902 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] avg=131072.0000 max=132385 max_rank=6 min=127082 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60397 max_load_reduction_pct=0.313292 moved_to_replicas=232902 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[78240, 130772, 146679, 66069, 141701, 165251, 192782, 127082] avg=131072.0000 max=192782 max_rank=6 min=66069 min_rank=3 imbalance_ratio=1.470810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[163269, 181444, 98305, 119674, 162580, 97957, 89949, 135398] avg=131072.0000 max=181444 max_rank=1 min=89949 min_rank=6 imbalance_ratio=1.384308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] avg=131072.0000 max=132385 max_rank=6 min=127082 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60397 max_load_reduction_pct=0.313292 moved_to_replicas=232902 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[163269, 181444, 98305, 119674, 162580, 97957, 89949, 135398] avg=131072.0000 max=181444 max_rank=1 min=89949 min_rank=6 imbalance_ratio=1.384308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[179803, 229281, 98105, 88359, 41626, 117551, 168664, 125187] avg=131072.0000 max=229281 max_rank=1 min=41626 min_rank=4 imbalance_ratio=1.749275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] avg=131072.0000 max=132385 max_rank=6 min=127082 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60397 max_load_reduction_pct=0.313292 moved_to_replicas=232902 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[78240, 130772, 146679, 66069, 141701, 165251, 192782, 127082] avg=131072.0000 max=192782 max_rank=6 min=66069 min_rank=3 imbalance_ratio=1.470810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] avg=131072.0000 max=132385 max_rank=1 min=125730 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=49059 max_load_reduction_pct=0.270381 moved_to_replicas=228723 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[163269, 181444, 98305, 119674, 162580, 97957, 89949, 135398] avg=131072.0000 max=181444 max_rank=1 min=89949 min_rank=6 imbalance_ratio=1.384308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] avg=131072.0000 max=132385 max_rank=1 min=125730 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=49059 max_load_reduction_pct=0.270381 moved_to_replicas=228723 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[163269, 181444, 98305, 119674, 162580, 97957, 89949, 135398] avg=131072.0000 max=181444 max_rank=1 min=89949 min_rank=6 imbalance_ratio=1.384308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[163269, 181444, 98305, 119674, 162580, 97957, 89949, 135398] avg=131072.0000 max=181444 max_rank=1 min=89949 min_rank=6 imbalance_ratio=1.384308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] avg=131072.0000 max=132385 max_rank=3 min=127089 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=96896 max_load_reduction_pct=0.422608 moved_to_replicas=365303 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[129599, 130772, 132383, 131589, 132383, 132383, 132385, 127082] avg=131072.0000 max=132385 max_rank=6 min=127082 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60397 max_load_reduction_pct=0.313292 moved_to_replicas=232902 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] avg=131072.0000 max=132385 max_rank=1 min=125730 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=49059 max_load_reduction_pct=0.270381 moved_to_replicas=228723 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] avg=131072.0000 max=132385 max_rank=1 min=125730 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=49059 max_load_reduction_pct=0.270381 moved_to_replicas=228723 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[163269, 181444, 98305, 119674, 162580, 97957, 89949, 135398] avg=131072.0000 max=181444 max_rank=1 min=89949 min_rank=6 imbalance_ratio=1.384308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[179803, 229281, 98105, 88359, 41626, 117551, 168664, 125187] avg=131072.0000 max=229281 max_rank=1 min=41626 min_rank=4 imbalance_ratio=1.749275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[179803, 229281, 98105, 88359, 41626, 117551, 168664, 125187] avg=131072.0000 max=229281 max_rank=1 min=41626 min_rank=4 imbalance_ratio=1.749275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] avg=131072.0000 max=132385 max_rank=1 min=125730 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=49059 max_load_reduction_pct=0.270381 moved_to_replicas=228723 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[163843, 32802, 98311, 141466, 140765, 118731, 130479, 222179] avg=131072.0000 max=222179 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.695091 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] avg=131072.0000 max=132385 max_rank=3 min=127089 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=96896 max_load_reduction_pct=0.422608 moved_to_replicas=365303 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] avg=131072.0000 max=132385 max_rank=1 min=125730 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=49059 max_load_reduction_pct=0.270381 moved_to_replicas=228723 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[179803, 229281, 98105, 88359, 41626, 117551, 168664, 125187] avg=131072.0000 max=229281 max_rank=1 min=41626 min_rank=4 imbalance_ratio=1.749275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[163269, 181444, 98305, 119674, 162580, 97957, 89949, 135398] avg=131072.0000 max=181444 max_rank=1 min=89949 min_rank=6 imbalance_ratio=1.384308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] avg=131072.0000 max=132385 max_rank=3 min=127089 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=96896 max_load_reduction_pct=0.422608 moved_to_replicas=365303 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[179803, 229281, 98105, 88359, 41626, 117551, 168664, 125187] avg=131072.0000 max=229281 max_rank=1 min=41626 min_rank=4 imbalance_ratio=1.749275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[179803, 229281, 98105, 88359, 41626, 117551, 168664, 125187] avg=131072.0000 max=229281 max_rank=1 min=41626 min_rank=4 imbalance_ratio=1.749275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] avg=131072.0000 max=132384 max_rank=3 min=127813 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=89795 max_load_reduction_pct=0.404156 moved_to_replicas=274952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] avg=131072.0000 max=132385 max_rank=3 min=127089 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=96896 max_load_reduction_pct=0.422608 moved_to_replicas=365303 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132385, 129191, 131742, 132383, 132379, 125730, 132383] avg=131072.0000 max=132385 max_rank=1 min=125730 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=49059 max_load_reduction_pct=0.270381 moved_to_replicas=228723 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] avg=131072.0000 max=132385 max_rank=3 min=127089 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=96896 max_load_reduction_pct=0.422608 moved_to_replicas=365303 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[163843, 32802, 98311, 141466, 140765, 118731, 130479, 222179] avg=131072.0000 max=222179 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.695091 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[179803, 229281, 98105, 88359, 41626, 117551, 168664, 125187] avg=131072.0000 max=229281 max_rank=1 min=41626 min_rank=4 imbalance_ratio=1.749275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] avg=131072.0000 max=132385 max_rank=3 min=127089 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=96896 max_load_reduction_pct=0.422608 moved_to_replicas=365303 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[163843, 32802, 98311, 141466, 140765, 118731, 130479, 222179] avg=131072.0000 max=222179 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.695091 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[98048, 163447, 130835, 59699, 65500, 234663, 163805, 132579] avg=131072.0000 max=234663 max_rank=5 min=59699 min_rank=3 imbalance_ratio=1.790337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] avg=131072.0000 max=132384 max_rank=3 min=127813 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=89795 max_load_reduction_pct=0.404156 moved_to_replicas=274952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] avg=131072.0000 max=132385 max_rank=3 min=127089 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=96896 max_load_reduction_pct=0.422608 moved_to_replicas=365303 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[163843, 32802, 98311, 141466, 140765, 118731, 130479, 222179] avg=131072.0000 max=222179 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.695091 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[163843, 32802, 98311, 141466, 140765, 118731, 130479, 222179] avg=131072.0000 max=222179 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.695091 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] avg=131072.0000 max=132384 max_rank=3 min=127813 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=89795 max_load_reduction_pct=0.404156 moved_to_replicas=274952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[179803, 229281, 98105, 88359, 41626, 117551, 168664, 125187] avg=131072.0000 max=229281 max_rank=1 min=41626 min_rank=4 imbalance_ratio=1.749275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126138 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=102280 max_load_reduction_pct=0.435859 moved_to_replicas=325309 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[163843, 32802, 98311, 141466, 140765, 118731, 130479, 222179] avg=131072.0000 max=222179 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.695091 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] avg=131072.0000 max=132384 max_rank=3 min=127813 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=89795 max_load_reduction_pct=0.404156 moved_to_replicas=274952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[98048, 163447, 130835, 59699, 65500, 234663, 163805, 132579] avg=131072.0000 max=234663 max_rank=5 min=59699 min_rank=3 imbalance_ratio=1.790337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] avg=131072.0000 max=132384 max_rank=3 min=127813 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=89795 max_load_reduction_pct=0.404156 moved_to_replicas=274952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[132384, 132383, 130869, 132385, 127089, 132381, 132383, 128702] avg=131072.0000 max=132385 max_rank=3 min=127089 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=96896 max_load_reduction_pct=0.422608 moved_to_replicas=365303 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[163843, 32802, 98311, 141466, 140765, 118731, 130479, 222179] avg=131072.0000 max=222179 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.695091 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] avg=131072.0000 max=132384 max_rank=3 min=127813 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=89795 max_load_reduction_pct=0.404156 moved_to_replicas=274952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[98048, 163447, 130835, 59699, 65500, 234663, 163805, 132579] avg=131072.0000 max=234663 max_rank=5 min=59699 min_rank=3 imbalance_ratio=1.790337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[168022, 229464, 132543, 98314, 99402, 96118, 126396, 98317] avg=131072.0000 max=229464 max_rank=1 min=96118 min_rank=5 imbalance_ratio=1.750671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126138 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=102280 max_load_reduction_pct=0.435859 moved_to_replicas=325309 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] avg=131072.0000 max=132384 max_rank=3 min=127813 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=89795 max_load_reduction_pct=0.404156 moved_to_replicas=274952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[98048, 163447, 130835, 59699, 65500, 234663, 163805, 132579] avg=131072.0000 max=234663 max_rank=5 min=59699 min_rank=3 imbalance_ratio=1.790337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126138 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=102280 max_load_reduction_pct=0.435859 moved_to_replicas=325309 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[98048, 163447, 130835, 59699, 65500, 234663, 163805, 132579] avg=131072.0000 max=234663 max_rank=5 min=59699 min_rank=3 imbalance_ratio=1.790337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] avg=131072.0000 max=132383 max_rank=0 min=129044 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97081 max_load_reduction_pct=0.423077 moved_to_replicas=266077 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[163843, 32802, 98311, 141466, 140765, 118731, 130479, 222179] avg=131072.0000 max=222179 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.695091 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[98048, 163447, 130835, 59699, 65500, 234663, 163805, 132579] avg=131072.0000 max=234663 max_rank=5 min=59699 min_rank=3 imbalance_ratio=1.790337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126138 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=102280 max_load_reduction_pct=0.435859 moved_to_replicas=325309 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[168022, 229464, 132543, 98314, 99402, 96118, 126396, 98317] avg=131072.0000 max=229464 max_rank=1 min=96118 min_rank=5 imbalance_ratio=1.750671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126138 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=102280 max_load_reduction_pct=0.435859 moved_to_replicas=325309 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[98048, 163447, 130835, 59699, 65500, 234663, 163805, 132579] avg=131072.0000 max=234663 max_rank=5 min=59699 min_rank=3 imbalance_ratio=1.790337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129795, 130956, 132384, 132382, 127813, 130479, 132384] avg=131072.0000 max=132384 max_rank=3 min=127813 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=89795 max_load_reduction_pct=0.404156 moved_to_replicas=274952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[168022, 229464, 132543, 98314, 99402, 96118, 126396, 98317] avg=131072.0000 max=229464 max_rank=1 min=96118 min_rank=5 imbalance_ratio=1.750671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126138 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=102280 max_load_reduction_pct=0.435859 moved_to_replicas=325309 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[158870, 131336, 196142, 98805, 196608, 67202, 166830, 32783] avg=131072.0000 max=196608 max_rank=4 min=32783 min_rank=7 imbalance_ratio=1.500000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] avg=131072.0000 max=132383 max_rank=0 min=129044 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97081 max_load_reduction_pct=0.423077 moved_to_replicas=266077 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126138 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=102280 max_load_reduction_pct=0.435859 moved_to_replicas=325309 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[168022, 229464, 132543, 98314, 99402, 96118, 126396, 98317] avg=131072.0000 max=229464 max_rank=1 min=96118 min_rank=5 imbalance_ratio=1.750671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] avg=131072.0000 max=132383 max_rank=0 min=129044 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97081 max_load_reduction_pct=0.423077 moved_to_replicas=266077 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] avg=131072.0000 max=132383 max_rank=0 min=128001 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64225 max_load_reduction_pct=0.326665 moved_to_replicas=378479 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[168022, 229464, 132543, 98314, 99402, 96118, 126396, 98317] avg=131072.0000 max=229464 max_rank=1 min=96118 min_rank=5 imbalance_ratio=1.750671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[98048, 163447, 130835, 59699, 65500, 234663, 163805, 132579] avg=131072.0000 max=234663 max_rank=5 min=59699 min_rank=3 imbalance_ratio=1.790337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[168022, 229464, 132543, 98314, 99402, 96118, 126396, 98317] avg=131072.0000 max=229464 max_rank=1 min=96118 min_rank=5 imbalance_ratio=1.750671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[158870, 131336, 196142, 98805, 196608, 67202, 166830, 32783] avg=131072.0000 max=196608 max_rank=4 min=32783 min_rank=7 imbalance_ratio=1.500000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] avg=131072.0000 max=132383 max_rank=0 min=129044 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97081 max_load_reduction_pct=0.423077 moved_to_replicas=266077 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] avg=131072.0000 max=132383 max_rank=0 min=129044 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97081 max_load_reduction_pct=0.423077 moved_to_replicas=266077 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[168022, 229464, 132543, 98314, 99402, 96118, 126396, 98317] avg=131072.0000 max=229464 max_rank=1 min=96118 min_rank=5 imbalance_ratio=1.750671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[158870, 131336, 196142, 98805, 196608, 67202, 166830, 32783] avg=131072.0000 max=196608 max_rank=4 min=32783 min_rank=7 imbalance_ratio=1.500000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[132980, 147743, 132109, 163864, 65579, 99891, 130979, 175431] avg=131072.0000 max=175431 max_rank=7 min=65579 min_rank=4 imbalance_ratio=1.338432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130835, 126138, 129688, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126138 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=102280 max_load_reduction_pct=0.435859 moved_to_replicas=325309 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] avg=131072.0000 max=132383 max_rank=0 min=129044 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97081 max_load_reduction_pct=0.423077 moved_to_replicas=266077 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] avg=131072.0000 max=132383 max_rank=0 min=128001 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64225 max_load_reduction_pct=0.326665 moved_to_replicas=378479 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] avg=131072.0000 max=132383 max_rank=0 min=129044 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97081 max_load_reduction_pct=0.423077 moved_to_replicas=266077 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] avg=131072.0000 max=132383 max_rank=0 min=128001 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64225 max_load_reduction_pct=0.326665 moved_to_replicas=378479 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[158870, 131336, 196142, 98805, 196608, 67202, 166830, 32783] avg=131072.0000 max=196608 max_rank=4 min=32783 min_rank=7 imbalance_ratio=1.500000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] avg=131072.0000 max=132385 max_rank=1 min=124582 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=43046 max_load_reduction_pct=0.245373 moved_to_replicas=179360 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[158870, 131336, 196142, 98805, 196608, 67202, 166830, 32783] avg=131072.0000 max=196608 max_rank=4 min=32783 min_rank=7 imbalance_ratio=1.500000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[168022, 229464, 132543, 98314, 99402, 96118, 126396, 98317] avg=131072.0000 max=229464 max_rank=1 min=96118 min_rank=5 imbalance_ratio=1.750671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[132980, 147743, 132109, 163864, 65579, 99891, 130979, 175431] avg=131072.0000 max=175431 max_rank=7 min=65579 min_rank=4 imbalance_ratio=1.338432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[158870, 131336, 196142, 98805, 196608, 67202, 166830, 32783] avg=131072.0000 max=196608 max_rank=4 min=32783 min_rank=7 imbalance_ratio=1.500000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] avg=131072.0000 max=132383 max_rank=0 min=128001 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64225 max_load_reduction_pct=0.326665 moved_to_replicas=378479 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] avg=131072.0000 max=132383 max_rank=0 min=128001 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64225 max_load_reduction_pct=0.326665 moved_to_replicas=378479 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[158870, 131336, 196142, 98805, 196608, 67202, 166830, 32783] avg=131072.0000 max=196608 max_rank=4 min=32783 min_rank=7 imbalance_ratio=1.500000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[132980, 147743, 132109, 163864, 65579, 99891, 130979, 175431] avg=131072.0000 max=175431 max_rank=7 min=65579 min_rank=4 imbalance_ratio=1.338432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[98209, 231122, 196431, 65557, 131280, 65550, 96794, 163633] avg=131072.0000 max=231122 max_rank=1 min=65550 min_rank=5 imbalance_ratio=1.763321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132168, 129044, 129269, 129867] avg=131072.0000 max=132383 max_rank=0 min=129044 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97081 max_load_reduction_pct=0.423077 moved_to_replicas=266077 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] avg=131072.0000 max=132385 max_rank=1 min=124582 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=43046 max_load_reduction_pct=0.245373 moved_to_replicas=179360 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] avg=131072.0000 max=132383 max_rank=0 min=128001 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64225 max_load_reduction_pct=0.326665 moved_to_replicas=378479 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] avg=131072.0000 max=132383 max_rank=0 min=128001 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64225 max_load_reduction_pct=0.326665 moved_to_replicas=378479 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] avg=131072.0000 max=132383 max_rank=1 min=129559 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98739 max_load_reduction_pct=0.427216 moved_to_replicas=387181 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] avg=131072.0000 max=132385 max_rank=1 min=124582 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=43046 max_load_reduction_pct=0.245373 moved_to_replicas=179360 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[132980, 147743, 132109, 163864, 65579, 99891, 130979, 175431] avg=131072.0000 max=175431 max_rank=7 min=65579 min_rank=4 imbalance_ratio=1.338432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[132980, 147743, 132109, 163864, 65579, 99891, 130979, 175431] avg=131072.0000 max=175431 max_rank=7 min=65579 min_rank=4 imbalance_ratio=1.338432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[98209, 231122, 196431, 65557, 131280, 65550, 96794, 163633] avg=131072.0000 max=231122 max_rank=1 min=65550 min_rank=5 imbalance_ratio=1.763321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[158870, 131336, 196142, 98805, 196608, 67202, 166830, 32783] avg=131072.0000 max=196608 max_rank=4 min=32783 min_rank=7 imbalance_ratio=1.500000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[132980, 147743, 132109, 163864, 65579, 99891, 130979, 175431] avg=131072.0000 max=175431 max_rank=7 min=65579 min_rank=4 imbalance_ratio=1.338432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] avg=131072.0000 max=132385 max_rank=1 min=124582 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=43046 max_load_reduction_pct=0.245373 moved_to_replicas=179360 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[132980, 147743, 132109, 163864, 65579, 99891, 130979, 175431] avg=131072.0000 max=175431 max_rank=7 min=65579 min_rank=4 imbalance_ratio=1.338432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] avg=131072.0000 max=132385 max_rank=1 min=124582 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=43046 max_load_reduction_pct=0.245373 moved_to_replicas=179360 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65662, 66013, 163860, 131057, 164106, 130650, 130580] avg=131072.0000 max=196648 max_rank=0 min=65662 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[98209, 231122, 196431, 65557, 131280, 65550, 96794, 163633] avg=131072.0000 max=231122 max_rank=1 min=65550 min_rank=5 imbalance_ratio=1.763321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] avg=131072.0000 max=132383 max_rank=1 min=129559 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98739 max_load_reduction_pct=0.427216 moved_to_replicas=387181 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[132383, 131336, 132383, 131571, 132383, 128136, 132383, 128001] avg=131072.0000 max=132383 max_rank=0 min=128001 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64225 max_load_reduction_pct=0.326665 moved_to_replicas=378479 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] avg=131072.0000 max=132385 max_rank=1 min=124582 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=43046 max_load_reduction_pct=0.245373 moved_to_replicas=179360 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] avg=131072.0000 max=132385 max_rank=1 min=124582 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=43046 max_load_reduction_pct=0.245373 moved_to_replicas=179360 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] avg=131072.0000 max=132383 max_rank=0 min=129235 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254669 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] avg=131072.0000 max=132383 max_rank=1 min=129559 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98739 max_load_reduction_pct=0.427216 moved_to_replicas=387181 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[98209, 231122, 196431, 65557, 131280, 65550, 96794, 163633] avg=131072.0000 max=231122 max_rank=1 min=65550 min_rank=5 imbalance_ratio=1.763321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[98209, 231122, 196431, 65557, 131280, 65550, 96794, 163633] avg=131072.0000 max=231122 max_rank=1 min=65550 min_rank=5 imbalance_ratio=1.763321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65662, 66013, 163860, 131057, 164106, 130650, 130580] avg=131072.0000 max=196648 max_rank=0 min=65662 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[132980, 147743, 132109, 163864, 65579, 99891, 130979, 175431] avg=131072.0000 max=175431 max_rank=7 min=65579 min_rank=4 imbalance_ratio=1.338432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[98209, 231122, 196431, 65557, 131280, 65550, 96794, 163633] avg=131072.0000 max=231122 max_rank=1 min=65550 min_rank=5 imbalance_ratio=1.763321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] avg=131072.0000 max=132383 max_rank=1 min=129559 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98739 max_load_reduction_pct=0.427216 moved_to_replicas=387181 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[131683, 98354, 130754, 162955, 163739, 130746, 67913, 162432] avg=131072.0000 max=163739 max_rank=4 min=67913 min_rank=6 imbalance_ratio=1.249229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] avg=131072.0000 max=132383 max_rank=1 min=129559 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98739 max_load_reduction_pct=0.427216 moved_to_replicas=387181 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[98209, 231122, 196431, 65557, 131280, 65550, 96794, 163633] avg=131072.0000 max=231122 max_rank=1 min=65550 min_rank=5 imbalance_ratio=1.763321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65662, 66013, 163860, 131057, 164106, 130650, 130580] avg=131072.0000 max=196648 max_rank=0 min=65662 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] avg=131072.0000 max=132383 max_rank=0 min=129235 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254669 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132385, 132109, 132383, 124582, 131372, 130979, 132383] avg=131072.0000 max=132385 max_rank=1 min=124582 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=43046 max_load_reduction_pct=0.245373 moved_to_replicas=179360 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] avg=131072.0000 max=132383 max_rank=1 min=129559 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98739 max_load_reduction_pct=0.427216 moved_to_replicas=387181 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] avg=131072.0000 max=132383 max_rank=3 min=128926 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31356 max_load_reduction_pct=0.191500 moved_to_replicas=182708 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] avg=131072.0000 max=132383 max_rank=1 min=129559 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98739 max_load_reduction_pct=0.427216 moved_to_replicas=387181 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] avg=131072.0000 max=132383 max_rank=0 min=129235 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254669 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65662, 66013, 163860, 131057, 164106, 130650, 130580] avg=131072.0000 max=196648 max_rank=0 min=65662 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65662, 66013, 163860, 131057, 164106, 130650, 130580] avg=131072.0000 max=196648 max_rank=0 min=65662 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[131683, 98354, 130754, 162955, 163739, 130746, 67913, 162432] avg=131072.0000 max=163739 max_rank=4 min=67913 min_rank=6 imbalance_ratio=1.249229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] avg=131072.0000 max=132383 max_rank=0 min=129235 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254669 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65662, 66013, 163860, 131057, 164106, 130650, 130580] avg=131072.0000 max=196648 max_rank=0 min=65662 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[98209, 231122, 196431, 65557, 131280, 65550, 96794, 163633] avg=131072.0000 max=231122 max_rank=1 min=65550 min_rank=5 imbalance_ratio=1.763321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[163845, 98343, 98305, 163803, 163820, 65473, 163301, 131686] avg=131072.0000 max=163845 max_rank=0 min=65473 min_rank=5 imbalance_ratio=1.250038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] avg=131072.0000 max=132383 max_rank=0 min=129235 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254669 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[131683, 98354, 130754, 162955, 163739, 130746, 67913, 162432] avg=131072.0000 max=163739 max_rank=4 min=67913 min_rank=6 imbalance_ratio=1.249229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65662, 66013, 163860, 131057, 164106, 130650, 130580] avg=131072.0000 max=196648 max_rank=0 min=65662 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] avg=131072.0000 max=132383 max_rank=3 min=128926 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31356 max_load_reduction_pct=0.191500 moved_to_replicas=182708 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] avg=131072.0000 max=132383 max_rank=0 min=129235 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254669 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[129901, 132383, 132383, 129605, 131280, 131082, 129559, 132383] avg=131072.0000 max=132383 max_rank=1 min=129559 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98739 max_load_reduction_pct=0.427216 moved_to_replicas=387181 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] avg=131072.0000 max=132383 max_rank=0 min=128372 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31462 max_load_reduction_pct=0.192023 moved_to_replicas=249645 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] avg=131072.0000 max=132383 max_rank=3 min=128926 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31356 max_load_reduction_pct=0.191500 moved_to_replicas=182708 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[131683, 98354, 130754, 162955, 163739, 130746, 67913, 162432] avg=131072.0000 max=163739 max_rank=4 min=67913 min_rank=6 imbalance_ratio=1.249229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] avg=131072.0000 max=132383 max_rank=0 min=129235 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254669 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[131683, 98354, 130754, 162955, 163739, 130746, 67913, 162432] avg=131072.0000 max=163739 max_rank=4 min=67913 min_rank=6 imbalance_ratio=1.249229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[163845, 98343, 98305, 163803, 163820, 65473, 163301, 131686] avg=131072.0000 max=163845 max_rank=0 min=65473 min_rank=5 imbalance_ratio=1.250038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] avg=131072.0000 max=132383 max_rank=3 min=128926 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31356 max_load_reduction_pct=0.191500 moved_to_replicas=182708 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[131683, 98354, 130754, 162955, 163739, 130746, 67913, 162432] avg=131072.0000 max=163739 max_rank=4 min=67913 min_rank=6 imbalance_ratio=1.249229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65662, 66013, 163860, 131057, 164106, 130650, 130580] avg=131072.0000 max=196648 max_rank=0 min=65662 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[228665, 98009, 132090, 195904, 65713, 130932, 737, 196526] avg=131072.0000 max=228665 max_rank=0 min=737 min_rank=6 imbalance_ratio=1.744576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[163845, 98343, 98305, 163803, 163820, 65473, 163301, 131686] avg=131072.0000 max=163845 max_rank=0 min=65473 min_rank=5 imbalance_ratio=1.250038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] avg=131072.0000 max=132383 max_rank=3 min=128926 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31356 max_load_reduction_pct=0.191500 moved_to_replicas=182708 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[131683, 98354, 130754, 162955, 163739, 130746, 67913, 162432] avg=131072.0000 max=163739 max_rank=4 min=67913 min_rank=6 imbalance_ratio=1.249229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] avg=131072.0000 max=132383 max_rank=0 min=128372 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31462 max_load_reduction_pct=0.192023 moved_to_replicas=249645 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] avg=131072.0000 max=132383 max_rank=3 min=128926 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31356 max_load_reduction_pct=0.191500 moved_to_replicas=182708 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129235, 132383, 131057, 132383, 130650, 130580] avg=131072.0000 max=132383 max_rank=0 min=129235 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254669 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] avg=131072.0000 max=132383 max_rank=0 min=127840 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96282 max_load_reduction_pct=0.421061 moved_to_replicas=444829 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] avg=131072.0000 max=132383 max_rank=0 min=128372 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31462 max_load_reduction_pct=0.192023 moved_to_replicas=249645 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[163845, 98343, 98305, 163803, 163820, 65473, 163301, 131686] avg=131072.0000 max=163845 max_rank=0 min=65473 min_rank=5 imbalance_ratio=1.250038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] avg=131072.0000 max=132383 max_rank=3 min=128926 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31356 max_load_reduction_pct=0.191500 moved_to_replicas=182708 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[163845, 98343, 98305, 163803, 163820, 65473, 163301, 131686] avg=131072.0000 max=163845 max_rank=0 min=65473 min_rank=5 imbalance_ratio=1.250038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[228665, 98009, 132090, 195904, 65713, 130932, 737, 196526] avg=131072.0000 max=228665 max_rank=0 min=737 min_rank=6 imbalance_ratio=1.744576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] avg=131072.0000 max=132383 max_rank=0 min=128372 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31462 max_load_reduction_pct=0.192023 moved_to_replicas=249645 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[163845, 98343, 98305, 163803, 163820, 65473, 163301, 131686] avg=131072.0000 max=163845 max_rank=0 min=65473 min_rank=5 imbalance_ratio=1.250038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[131683, 98354, 130754, 162955, 163739, 130746, 67913, 162432] avg=131072.0000 max=163739 max_rank=4 min=67913 min_rank=6 imbalance_ratio=1.249229 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[228665, 98009, 132090, 195904, 65713, 130932, 737, 196526] avg=131072.0000 max=228665 max_rank=0 min=737 min_rank=6 imbalance_ratio=1.744576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] avg=131072.0000 max=132383 max_rank=0 min=128372 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31462 max_load_reduction_pct=0.192023 moved_to_replicas=249645 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[163845, 98343, 98305, 163803, 163820, 65473, 163301, 131686] avg=131072.0000 max=163845 max_rank=0 min=65473 min_rank=5 imbalance_ratio=1.250038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] avg=131072.0000 max=132383 max_rank=0 min=127840 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96282 max_load_reduction_pct=0.421061 moved_to_replicas=444829 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] avg=131072.0000 max=132383 max_rank=0 min=128372 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31462 max_load_reduction_pct=0.192023 moved_to_replicas=249645 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[131683, 128926, 130754, 132383, 132383, 130746, 129318, 132383] avg=131072.0000 max=132383 max_rank=3 min=128926 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31356 max_load_reduction_pct=0.191500 moved_to_replicas=182708 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] avg=131072.0000 max=132383 max_rank=0 min=127840 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96282 max_load_reduction_pct=0.421061 moved_to_replicas=444829 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[228665, 98009, 132090, 195904, 65713, 130932, 737, 196526] avg=131072.0000 max=228665 max_rank=0 min=737 min_rank=6 imbalance_ratio=1.744576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] avg=131072.0000 max=132383 max_rank=0 min=128372 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31462 max_load_reduction_pct=0.192023 moved_to_replicas=249645 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[228665, 98009, 132090, 195904, 65713, 130932, 737, 196526] avg=131072.0000 max=228665 max_rank=0 min=737 min_rank=6 imbalance_ratio=1.744576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] avg=131072.0000 max=132383 max_rank=0 min=127840 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96282 max_load_reduction_pct=0.421061 moved_to_replicas=444829 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[228665, 98009, 132090, 195904, 65713, 130932, 737, 196526] avg=131072.0000 max=228665 max_rank=0 min=737 min_rank=6 imbalance_ratio=1.744576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[163845, 98343, 98305, 163803, 163820, 65473, 163301, 131686] avg=131072.0000 max=163845 max_rank=0 min=65473 min_rank=5 imbalance_ratio=1.250038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] avg=131072.0000 max=132383 max_rank=0 min=127840 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96282 max_load_reduction_pct=0.421061 moved_to_replicas=444829 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[228665, 98009, 132090, 195904, 65713, 130932, 737, 196526] avg=131072.0000 max=228665 max_rank=0 min=737 min_rank=6 imbalance_ratio=1.744576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] avg=131072.0000 max=132383 max_rank=0 min=127840 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96282 max_load_reduction_pct=0.421061 moved_to_replicas=444829 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129261, 129725, 132383, 132383, 128372, 132383, 131686] avg=131072.0000 max=132383 max_rank=0 min=128372 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31462 max_load_reduction_pct=0.192023 moved_to_replicas=249645 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] avg=131072.0000 max=132383 max_rank=0 min=127840 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96282 max_load_reduction_pct=0.421061 moved_to_replicas=444829 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[228665, 98009, 132090, 195904, 65713, 130932, 737, 196526] avg=131072.0000 max=228665 max_rank=0 min=737 min_rank=6 imbalance_ratio=1.744576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130775, 132090, 132383, 127840, 130932, 129790, 132383] avg=131072.0000 max=132383 max_rank=0 min=127840 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96282 max_load_reduction_pct=0.421061 moved_to_replicas=444829 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=4194089 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364544 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=4205685 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=353024 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=4189714 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368640 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=4191541 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=366080 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=4195343 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=362752 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=4177446 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=381440 peak_demand=132383 required_capacity=1.010002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=4220588 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=338560 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=4180026 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379520 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:19:09] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 561.7 | throughput per GPU (TFLOP/s/GPU): 593.6 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.033467E+01 | loss scale: 1.0 | grad norm: 17.376 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=143616 expected_recv=255334 expected_overflow=0 exact_recv_per_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=143616 expected_recv=260894 expected_overflow=0 exact_recv_per_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=143616 expected_recv=255334 actual_valid_recv=258048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=143616 expected_recv=257095 expected_overflow=0 exact_recv_per_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=258048 slack_slots=29184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=143616 expected_recv=260894 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=143616 expected_recv=257095 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=259456 slack_slots=27776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=143616 expected_recv=260773 expected_overflow=0 exact_recv_per_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=143616 expected_recv=256129 expected_overflow=0 exact_recv_per_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=143616 expected_recv=260773 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=143616 expected_recv=256129 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=259328 slack_slots=27904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=143616 expected_recv=258292 expected_overflow=0 exact_recv_per_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=143616 expected_recv=262892 expected_overflow=0 exact_recv_per_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=143616 expected_recv=258292 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=143616 expected_recv=262892 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=261120 slack_slots=26112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=143616 expected_recv=262460 expected_overflow=0 exact_recv_per_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=143616 expected_recv=262460 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=143616 expected_recv=257548 expected_overflow=0 exact_recv_per_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=143616 expected_recv=256005 expected_overflow=0 exact_recv_per_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=143616 expected_recv=262078 expected_overflow=0 exact_recv_per_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=143616 expected_recv=257548 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=143616 expected_recv=256005 actual_valid_recv=258944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=143616 expected_recv=262078 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=260608 slack_slots=26624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=258944 slack_slots=28288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=264960 slack_slots=22272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=143616 expected_recv=258070 expected_overflow=0 exact_recv_per_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=143616 expected_recv=260023 expected_overflow=0 exact_recv_per_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=143616 expected_recv=258070 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=143616 expected_recv=260839 expected_overflow=0 exact_recv_per_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=143616 expected_recv=262118 expected_overflow=0 exact_recv_per_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=143616 expected_recv=261804 expected_overflow=0 exact_recv_per_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=143616 expected_recv=260023 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=143616 expected_recv=260839 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=143616 expected_recv=262118 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=143616 expected_recv=261804 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=143616 expected_recv=262339 expected_overflow=0 exact_recv_per_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=143616 expected_recv=262339 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=143616 expected_recv=253547 expected_overflow=0 exact_recv_per_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=143616 expected_recv=262354 expected_overflow=0 exact_recv_per_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=143616 expected_recv=253547 actual_valid_recv=256000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=143616 expected_recv=262354 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=143616 expected_recv=261339 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=256000 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=143616 expected_recv=261339 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=143616 expected_recv=259848 expected_overflow=0 exact_recv_per_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=143616 expected_recv=259848 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=143616 expected_recv=255825 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=143616 expected_recv=256541 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=143616 expected_recv=255825 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=143616 expected_recv=256541 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=258688 slack_slots=28544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=259584 slack_slots=27648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=143616 expected_recv=251289 expected_overflow=0 exact_recv_per_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=143616 expected_recv=264386 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=143616 expected_recv=251289 actual_valid_recv=254208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=143616 expected_recv=264386 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=254208 slack_slots=33024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=143616 expected_recv=257264 expected_overflow=0 exact_recv_per_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=143616 expected_recv=257264 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=143616 expected_recv=262022 expected_overflow=0 exact_recv_per_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=260992 slack_slots=26240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=268032 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=143616 expected_recv=262022 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=143616 expected_recv=260688 expected_overflow=0 exact_recv_per_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=143616 expected_recv=260590 expected_overflow=0 exact_recv_per_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=143616 expected_recv=259848 expected_overflow=0 exact_recv_per_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=143616 expected_recv=260688 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=143616 expected_recv=259705 expected_overflow=0 exact_recv_per_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=143616 expected_recv=260590 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=143616 expected_recv=259848 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=143616 expected_recv=261273 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=143616 expected_recv=259705 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=263424 slack_slots=23808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=143616 expected_recv=261273 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=143616 expected_recv=261365 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=143616 expected_recv=261365 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=143616 expected_recv=258107 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=143616 expected_recv=260023 expected_overflow=0 exact_recv_per_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=143616 expected_recv=258107 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=143616 expected_recv=260023 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=143616 expected_recv=259859 expected_overflow=0 exact_recv_per_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=260480 slack_slots=26752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=143616 expected_recv=259859 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=143616 expected_recv=257343 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=143616 expected_recv=257343 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=259712 slack_slots=27520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=143616 expected_recv=259429 expected_overflow=0 exact_recv_per_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=143616 expected_recv=258777 expected_overflow=0 exact_recv_per_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=143616 expected_recv=260515 expected_overflow=0 exact_recv_per_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=143616 expected_recv=259429 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=143616 expected_recv=258777 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=143616 expected_recv=261033 expected_overflow=0 exact_recv_per_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=143616 expected_recv=260515 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=143616 expected_recv=261033 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=143616 expected_recv=260548 expected_overflow=0 exact_recv_per_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=143616 expected_recv=261325 expected_overflow=0 exact_recv_per_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=143616 expected_recv=259433 expected_overflow=0 exact_recv_per_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=143616 expected_recv=260548 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=143616 expected_recv=261325 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=143616 expected_recv=259433 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=143616 expected_recv=262201 expected_overflow=0 exact_recv_per_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=143616 expected_recv=260339 expected_overflow=0 exact_recv_per_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=143616 expected_recv=262201 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=143616 expected_recv=260339 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=143616 expected_recv=259963 expected_overflow=0 exact_recv_per_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=143616 expected_recv=263740 expected_overflow=0 exact_recv_per_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=143616 expected_recv=259963 actual_valid_recv=262656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=143616 expected_recv=261927 expected_overflow=0 exact_recv_per_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=143616 expected_recv=262245 expected_overflow=0 exact_recv_per_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=262656 slack_slots=24576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=143616 expected_recv=263740 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=143616 expected_recv=261927 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=143616 expected_recv=262245 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=265600 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=143616 expected_recv=256904 expected_overflow=0 exact_recv_per_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=143616 expected_recv=261972 expected_overflow=0 exact_recv_per_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=143616 expected_recv=257252 expected_overflow=0 exact_recv_per_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=143616 expected_recv=256904 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=143616 expected_recv=261972 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=143616 expected_recv=257252 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=259840 slack_slots=27392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=259584 slack_slots=27648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=143616 expected_recv=262045 expected_overflow=0 exact_recv_per_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=143616 expected_recv=262045 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=143616 expected_recv=261887 expected_overflow=0 exact_recv_per_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=143616 expected_recv=259338 expected_overflow=0 exact_recv_per_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=143616 expected_recv=261887 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=143616 expected_recv=259338 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=143616 expected_recv=262220 expected_overflow=0 exact_recv_per_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=143616 expected_recv=260543 expected_overflow=0 exact_recv_per_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=143616 expected_recv=257396 expected_overflow=0 exact_recv_per_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=143616 expected_recv=262220 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=143616 expected_recv=260543 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=143616 expected_recv=259167 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=143616 expected_recv=263357 expected_overflow=0 exact_recv_per_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=143616 expected_recv=257396 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=143616 expected_recv=259167 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=143616 expected_recv=263357 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=259712 slack_slots=27520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=143616 expected_recv=256783 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=265216 slack_slots=22016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=143616 expected_recv=256783 actual_valid_recv=258944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=258944 slack_slots=28288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=143616 expected_recv=257370 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=143616 expected_recv=257370 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=143616 expected_recv=262130 expected_overflow=0 exact_recv_per_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=259968 slack_slots=27264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=143616 expected_recv=262527 expected_overflow=0 exact_recv_per_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=143616 expected_recv=262130 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=143616 expected_recv=259241 expected_overflow=0 exact_recv_per_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=143616 expected_recv=262527 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=143616 expected_recv=259241 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=143616 expected_recv=259726 expected_overflow=0 exact_recv_per_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=143616 expected_recv=259230 expected_overflow=0 exact_recv_per_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=143616 expected_recv=259726 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=143616 expected_recv=259230 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=261120 slack_slots=26112 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[83226, 119802, 143434, 140841, 117795, 188129, 150822, 104527] avg=131072.0000 max=188129 max_rank=5 min=83226 min_rank=0 imbalance_ratio=1.435310 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[83226, 119802, 143434, 140841, 117795, 188129, 150822, 104527] avg=131072.0000 max=188129 max_rank=5 min=83226 min_rank=0 imbalance_ratio=1.435310 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[83226, 119802, 143434, 140841, 117795, 188129, 150822, 104527] avg=131072.0000 max=188129 max_rank=5 min=83226 min_rank=0 imbalance_ratio=1.435310 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] avg=131072.0000 max=132383 max_rank=0 min=128260 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=55746 max_load_reduction_pct=0.296318 moved_to_replicas=189196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] avg=131072.0000 max=132383 max_rank=0 min=128260 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=55746 max_load_reduction_pct=0.296318 moved_to_replicas=189196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[83226, 119802, 143434, 140841, 117795, 188129, 150822, 104527] avg=131072.0000 max=188129 max_rank=5 min=83226 min_rank=0 imbalance_ratio=1.435310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[83226, 119802, 143434, 140841, 117795, 188129, 150822, 104527] avg=131072.0000 max=188129 max_rank=5 min=83226 min_rank=0 imbalance_ratio=1.435310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] avg=131072.0000 max=132383 max_rank=0 min=128260 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=55746 max_load_reduction_pct=0.296318 moved_to_replicas=189196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[83226, 119802, 143434, 140841, 117795, 188129, 150822, 104527] avg=131072.0000 max=188129 max_rank=5 min=83226 min_rank=0 imbalance_ratio=1.435310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[83226, 119802, 143434, 140841, 117795, 188129, 150822, 104527] avg=131072.0000 max=188129 max_rank=5 min=83226 min_rank=0 imbalance_ratio=1.435310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[193966, 98406, 196004, 130347, 16046, 147412, 169014, 97381] avg=131072.0000 max=196004 max_rank=2 min=16046 min_rank=4 imbalance_ratio=1.495392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] avg=131072.0000 max=132383 max_rank=0 min=128260 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=55746 max_load_reduction_pct=0.296318 moved_to_replicas=189196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] avg=131072.0000 max=132383 max_rank=0 min=128260 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=55746 max_load_reduction_pct=0.296318 moved_to_replicas=189196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[83226, 119802, 143434, 140841, 117795, 188129, 150822, 104527] avg=131072.0000 max=188129 max_rank=5 min=83226 min_rank=0 imbalance_ratio=1.435310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[193966, 98406, 196004, 130347, 16046, 147412, 169014, 97381] avg=131072.0000 max=196004 max_rank=2 min=16046 min_rank=4 imbalance_ratio=1.495392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] avg=131072.0000 max=132383 max_rank=0 min=128260 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=55746 max_load_reduction_pct=0.296318 moved_to_replicas=189196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[193966, 98406, 196004, 130347, 16046, 147412, 169014, 97381] avg=131072.0000 max=196004 max_rank=2 min=16046 min_rank=4 imbalance_ratio=1.495392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] avg=131072.0000 max=132384 max_rank=5 min=126202 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63620 max_load_reduction_pct=0.324585 moved_to_replicas=349162 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] avg=131072.0000 max=132383 max_rank=0 min=128260 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=55746 max_load_reduction_pct=0.296318 moved_to_replicas=189196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[132383, 128260, 132383, 132383, 129180, 132383, 132383, 129221] avg=131072.0000 max=132383 max_rank=0 min=128260 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=55746 max_load_reduction_pct=0.296318 moved_to_replicas=189196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] avg=131072.0000 max=132384 max_rank=5 min=126202 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63620 max_load_reduction_pct=0.324585 moved_to_replicas=349162 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[193966, 98406, 196004, 130347, 16046, 147412, 169014, 97381] avg=131072.0000 max=196004 max_rank=2 min=16046 min_rank=4 imbalance_ratio=1.495392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[193966, 98406, 196004, 130347, 16046, 147412, 169014, 97381] avg=131072.0000 max=196004 max_rank=2 min=16046 min_rank=4 imbalance_ratio=1.495392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] avg=131072.0000 max=132384 max_rank=5 min=126202 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63620 max_load_reduction_pct=0.324585 moved_to_replicas=349162 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[193966, 98406, 196004, 130347, 16046, 147412, 169014, 97381] avg=131072.0000 max=196004 max_rank=2 min=16046 min_rank=4 imbalance_ratio=1.495392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[104702, 131164, 154167, 145490, 98264, 170212, 195940, 48637] avg=131072.0000 max=195940 max_rank=6 min=48637 min_rank=7 imbalance_ratio=1.494904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[193966, 98406, 196004, 130347, 16046, 147412, 169014, 97381] avg=131072.0000 max=196004 max_rank=2 min=16046 min_rank=4 imbalance_ratio=1.495392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] avg=131072.0000 max=132384 max_rank=5 min=126202 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63620 max_load_reduction_pct=0.324585 moved_to_replicas=349162 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] avg=131072.0000 max=132384 max_rank=5 min=126202 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63620 max_load_reduction_pct=0.324585 moved_to_replicas=349162 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[104702, 131164, 154167, 145490, 98264, 170212, 195940, 48637] avg=131072.0000 max=195940 max_rank=6 min=48637 min_rank=7 imbalance_ratio=1.494904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[193966, 98406, 196004, 130347, 16046, 147412, 169014, 97381] avg=131072.0000 max=196004 max_rank=2 min=16046 min_rank=4 imbalance_ratio=1.495392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] avg=131072.0000 max=132384 max_rank=5 min=126202 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63620 max_load_reduction_pct=0.324585 moved_to_replicas=349162 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] avg=131072.0000 max=132385 max_rank=3 min=125299 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63555 max_load_reduction_pct=0.324359 moved_to_replicas=270715 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[104702, 131164, 154167, 145490, 98264, 170212, 195940, 48637] avg=131072.0000 max=195940 max_rank=6 min=48637 min_rank=7 imbalance_ratio=1.494904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] avg=131072.0000 max=132384 max_rank=5 min=126202 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63620 max_load_reduction_pct=0.324585 moved_to_replicas=349162 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] avg=131072.0000 max=132385 max_rank=3 min=125299 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63555 max_load_reduction_pct=0.324359 moved_to_replicas=270715 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131143, 132383, 130347, 131351, 132384, 132383, 126202] avg=131072.0000 max=132384 max_rank=5 min=126202 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63620 max_load_reduction_pct=0.324585 moved_to_replicas=349162 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[104702, 131164, 154167, 145490, 98264, 170212, 195940, 48637] avg=131072.0000 max=195940 max_rank=6 min=48637 min_rank=7 imbalance_ratio=1.494904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[104702, 131164, 154167, 145490, 98264, 170212, 195940, 48637] avg=131072.0000 max=195940 max_rank=6 min=48637 min_rank=7 imbalance_ratio=1.494904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] avg=131072.0000 max=132385 max_rank=3 min=125299 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63555 max_load_reduction_pct=0.324359 moved_to_replicas=270715 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[104702, 131164, 154167, 145490, 98264, 170212, 195940, 48637] avg=131072.0000 max=195940 max_rank=6 min=48637 min_rank=7 imbalance_ratio=1.494904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[97854, 98473, 65067, 196653, 99511, 196334, 163638, 131046] avg=131072.0000 max=196653 max_rank=3 min=65067 min_rank=2 imbalance_ratio=1.500343 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] avg=131072.0000 max=132385 max_rank=3 min=125299 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63555 max_load_reduction_pct=0.324359 moved_to_replicas=270715 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[104702, 131164, 154167, 145490, 98264, 170212, 195940, 48637] avg=131072.0000 max=195940 max_rank=6 min=48637 min_rank=7 imbalance_ratio=1.494904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[97854, 98473, 65067, 196653, 99511, 196334, 163638, 131046] avg=131072.0000 max=196653 max_rank=3 min=65067 min_rank=2 imbalance_ratio=1.500343 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] avg=131072.0000 max=132385 max_rank=3 min=125299 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63555 max_load_reduction_pct=0.324359 moved_to_replicas=270715 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[104702, 131164, 154167, 145490, 98264, 170212, 195940, 48637] avg=131072.0000 max=195940 max_rank=6 min=48637 min_rank=7 imbalance_ratio=1.494904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] avg=131072.0000 max=132385 max_rank=3 min=125299 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63555 max_load_reduction_pct=0.324359 moved_to_replicas=270715 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] avg=131072.0000 max=132383 max_rank=3 min=129337 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64270 max_load_reduction_pct=0.326819 moved_to_replicas=319027 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[97854, 98473, 65067, 196653, 99511, 196334, 163638, 131046] avg=131072.0000 max=196653 max_rank=3 min=65067 min_rank=2 imbalance_ratio=1.500343 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] avg=131072.0000 max=132383 max_rank=3 min=129337 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64270 max_load_reduction_pct=0.326819 moved_to_replicas=319027 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] avg=131072.0000 max=132385 max_rank=3 min=125299 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63555 max_load_reduction_pct=0.324359 moved_to_replicas=270715 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[131555, 131164, 132382, 132385, 131026, 132382, 132383, 125299] avg=131072.0000 max=132385 max_rank=3 min=125299 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63555 max_load_reduction_pct=0.324359 moved_to_replicas=270715 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[97854, 98473, 65067, 196653, 99511, 196334, 163638, 131046] avg=131072.0000 max=196653 max_rank=3 min=65067 min_rank=2 imbalance_ratio=1.500343 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[97854, 98473, 65067, 196653, 99511, 196334, 163638, 131046] avg=131072.0000 max=196653 max_rank=3 min=65067 min_rank=2 imbalance_ratio=1.500343 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] avg=131072.0000 max=132383 max_rank=3 min=129337 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64270 max_load_reduction_pct=0.326819 moved_to_replicas=319027 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65595, 227037, 131158, 157958, 179392, 122847, 66218, 98371] avg=131072.0000 max=227037 max_rank=1 min=65595 min_rank=0 imbalance_ratio=1.732155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[97854, 98473, 65067, 196653, 99511, 196334, 163638, 131046] avg=131072.0000 max=196653 max_rank=3 min=65067 min_rank=2 imbalance_ratio=1.500343 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65595, 227037, 131158, 157958, 179392, 122847, 66218, 98371] avg=131072.0000 max=227037 max_rank=1 min=65595 min_rank=0 imbalance_ratio=1.732155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] avg=131072.0000 max=132383 max_rank=3 min=129337 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64270 max_load_reduction_pct=0.326819 moved_to_replicas=319027 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[97854, 98473, 65067, 196653, 99511, 196334, 163638, 131046] avg=131072.0000 max=196653 max_rank=3 min=65067 min_rank=2 imbalance_ratio=1.500343 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] avg=131072.0000 max=132383 max_rank=3 min=129337 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64270 max_load_reduction_pct=0.326819 moved_to_replicas=319027 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[97854, 98473, 65067, 196653, 99511, 196334, 163638, 131046] avg=131072.0000 max=196653 max_rank=3 min=65067 min_rank=2 imbalance_ratio=1.500343 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] avg=131072.0000 max=132385 max_rank=4 min=127488 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=94652 max_load_reduction_pct=0.416901 moved_to_replicas=337478 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] avg=131072.0000 max=132383 max_rank=3 min=129337 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64270 max_load_reduction_pct=0.326819 moved_to_replicas=319027 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65595, 227037, 131158, 157958, 179392, 122847, 66218, 98371] avg=131072.0000 max=227037 max_rank=1 min=65595 min_rank=0 imbalance_ratio=1.732155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] avg=131072.0000 max=132385 max_rank=4 min=127488 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=94652 max_load_reduction_pct=0.416901 moved_to_replicas=337478 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] avg=131072.0000 max=132383 max_rank=3 min=129337 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64270 max_load_reduction_pct=0.326819 moved_to_replicas=319027 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[130617, 129661, 129337, 132383, 130766, 132383, 132383, 131046] avg=131072.0000 max=132383 max_rank=3 min=129337 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64270 max_load_reduction_pct=0.326819 moved_to_replicas=319027 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65595, 227037, 131158, 157958, 179392, 122847, 66218, 98371] avg=131072.0000 max=227037 max_rank=1 min=65595 min_rank=0 imbalance_ratio=1.732155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65595, 227037, 131158, 157958, 179392, 122847, 66218, 98371] avg=131072.0000 max=227037 max_rank=1 min=65595 min_rank=0 imbalance_ratio=1.732155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] avg=131072.0000 max=132385 max_rank=4 min=127488 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=94652 max_load_reduction_pct=0.416901 moved_to_replicas=337478 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[163828, 196558, 157953, 66437, 98553, 133784, 165895, 65568] avg=131072.0000 max=196558 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] avg=131072.0000 max=132385 max_rank=4 min=127488 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=94652 max_load_reduction_pct=0.416901 moved_to_replicas=337478 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65595, 227037, 131158, 157958, 179392, 122847, 66218, 98371] avg=131072.0000 max=227037 max_rank=1 min=65595 min_rank=0 imbalance_ratio=1.732155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[163828, 196558, 157953, 66437, 98553, 133784, 165895, 65568] avg=131072.0000 max=196558 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] avg=131072.0000 max=132385 max_rank=4 min=127488 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=94652 max_load_reduction_pct=0.416901 moved_to_replicas=337478 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65595, 227037, 131158, 157958, 179392, 122847, 66218, 98371] avg=131072.0000 max=227037 max_rank=1 min=65595 min_rank=0 imbalance_ratio=1.732155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65595, 227037, 131158, 157958, 179392, 122847, 66218, 98371] avg=131072.0000 max=227037 max_rank=1 min=65595 min_rank=0 imbalance_ratio=1.732155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] avg=131072.0000 max=132383 max_rank=0 min=126052 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64175 max_load_reduction_pct=0.326494 moved_to_replicas=312478 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] avg=131072.0000 max=132385 max_rank=4 min=127488 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=94652 max_load_reduction_pct=0.416901 moved_to_replicas=337478 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] avg=131072.0000 max=132383 max_rank=0 min=126052 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64175 max_load_reduction_pct=0.326494 moved_to_replicas=312478 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[163828, 196558, 157953, 66437, 98553, 133784, 165895, 65568] avg=131072.0000 max=196558 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[163828, 196558, 157953, 66437, 98553, 133784, 165895, 65568] avg=131072.0000 max=196558 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] avg=131072.0000 max=132385 max_rank=4 min=127488 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=94652 max_load_reduction_pct=0.416901 moved_to_replicas=337478 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[127488, 132383, 131158, 132383, 132385, 129267, 132380, 131132] avg=131072.0000 max=132385 max_rank=4 min=127488 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=94652 max_load_reduction_pct=0.416901 moved_to_replicas=337478 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[163828, 196558, 157953, 66437, 98553, 133784, 165895, 65568] avg=131072.0000 max=196558 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] avg=131072.0000 max=132383 max_rank=0 min=126052 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64175 max_load_reduction_pct=0.326494 moved_to_replicas=312478 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[163768, 32804, 79100, 166765, 196410, 140854, 41472, 227403] avg=131072.0000 max=227403 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.734947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] avg=131072.0000 max=132383 max_rank=0 min=126052 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64175 max_load_reduction_pct=0.326494 moved_to_replicas=312478 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[163768, 32804, 79100, 166765, 196410, 140854, 41472, 227403] avg=131072.0000 max=227403 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.734947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[163828, 196558, 157953, 66437, 98553, 133784, 165895, 65568] avg=131072.0000 max=196558 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[163828, 196558, 157953, 66437, 98553, 133784, 165895, 65568] avg=131072.0000 max=196558 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[163828, 196558, 157953, 66437, 98553, 133784, 165895, 65568] avg=131072.0000 max=196558 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499619 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] avg=131072.0000 max=132383 max_rank=0 min=126052 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64175 max_load_reduction_pct=0.326494 moved_to_replicas=312478 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] avg=131072.0000 max=132385 max_rank=5 min=126450 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=95018 max_load_reduction_pct=0.417840 moved_to_replicas=472065 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] avg=131072.0000 max=132385 max_rank=5 min=126450 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=95018 max_load_reduction_pct=0.417840 moved_to_replicas=472065 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] avg=131072.0000 max=132383 max_rank=0 min=126052 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64175 max_load_reduction_pct=0.326494 moved_to_replicas=312478 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[163768, 32804, 79100, 166765, 196410, 140854, 41472, 227403] avg=131072.0000 max=227403 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.734947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[163768, 32804, 79100, 166765, 196410, 140854, 41472, 227403] avg=131072.0000 max=227403 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.734947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] avg=131072.0000 max=132383 max_rank=0 min=126052 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64175 max_load_reduction_pct=0.326494 moved_to_replicas=312478 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132382, 130612, 129998, 132383, 132383, 126052] avg=131072.0000 max=132383 max_rank=0 min=126052 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64175 max_load_reduction_pct=0.326494 moved_to_replicas=312478 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[64380, 163921, 155343, 98292, 74703, 295153, 98447, 98337] avg=131072.0000 max=295153 max_rank=5 min=64380 min_rank=0 imbalance_ratio=2.251839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[163768, 32804, 79100, 166765, 196410, 140854, 41472, 227403] avg=131072.0000 max=227403 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.734947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] avg=131072.0000 max=132385 max_rank=5 min=126450 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=95018 max_load_reduction_pct=0.417840 moved_to_replicas=472065 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[64380, 163921, 155343, 98292, 74703, 295153, 98447, 98337] avg=131072.0000 max=295153 max_rank=5 min=64380 min_rank=0 imbalance_ratio=2.251839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] avg=131072.0000 max=132385 max_rank=5 min=126450 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=95018 max_load_reduction_pct=0.417840 moved_to_replicas=472065 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[163768, 32804, 79100, 166765, 196410, 140854, 41472, 227403] avg=131072.0000 max=227403 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.734947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] avg=131072.0000 max=132383 max_rank=1 min=129898 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=162770 max_load_reduction_pct=0.551477 moved_to_replicas=432671 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] avg=131072.0000 max=132385 max_rank=5 min=126450 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=95018 max_load_reduction_pct=0.417840 moved_to_replicas=472065 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[163768, 32804, 79100, 166765, 196410, 140854, 41472, 227403] avg=131072.0000 max=227403 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.734947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] avg=131072.0000 max=132383 max_rank=1 min=129898 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=162770 max_load_reduction_pct=0.551477 moved_to_replicas=432671 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[163768, 32804, 79100, 166765, 196410, 140854, 41472, 227403] avg=131072.0000 max=227403 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.734947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] avg=131072.0000 max=132385 max_rank=5 min=126450 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=95018 max_load_reduction_pct=0.417840 moved_to_replicas=472065 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[64380, 163921, 155343, 98292, 74703, 295153, 98447, 98337] avg=131072.0000 max=295153 max_rank=5 min=64380 min_rank=0 imbalance_ratio=2.251839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[64380, 163921, 155343, 98292, 74703, 295153, 98447, 98337] avg=131072.0000 max=295153 max_rank=5 min=64380 min_rank=0 imbalance_ratio=2.251839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] avg=131072.0000 max=132385 max_rank=5 min=126450 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=95018 max_load_reduction_pct=0.417840 moved_to_replicas=472065 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127824, 126450, 132383, 132383, 132385, 132385, 132383] avg=131072.0000 max=132385 max_rank=5 min=126450 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=95018 max_load_reduction_pct=0.417840 moved_to_replicas=472065 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[159510, 228357, 130914, 65729, 98408, 65564, 161437, 138657] avg=131072.0000 max=228357 max_rank=1 min=65564 min_rank=5 imbalance_ratio=1.742226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[64380, 163921, 155343, 98292, 74703, 295153, 98447, 98337] avg=131072.0000 max=295153 max_rank=5 min=64380 min_rank=0 imbalance_ratio=2.251839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[159510, 228357, 130914, 65729, 98408, 65564, 161437, 138657] avg=131072.0000 max=228357 max_rank=1 min=65564 min_rank=5 imbalance_ratio=1.742226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] avg=131072.0000 max=132383 max_rank=1 min=129898 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=162770 max_load_reduction_pct=0.551477 moved_to_replicas=432671 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] avg=131072.0000 max=132383 max_rank=1 min=129898 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=162770 max_load_reduction_pct=0.551477 moved_to_replicas=432671 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[64380, 163921, 155343, 98292, 74703, 295153, 98447, 98337] avg=131072.0000 max=295153 max_rank=5 min=64380 min_rank=0 imbalance_ratio=2.251839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127462 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95974 max_load_reduction_pct=0.420281 moved_to_replicas=317197 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] avg=131072.0000 max=132383 max_rank=1 min=129898 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=162770 max_load_reduction_pct=0.551477 moved_to_replicas=432671 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127462 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95974 max_load_reduction_pct=0.420281 moved_to_replicas=317197 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[64380, 163921, 155343, 98292, 74703, 295153, 98447, 98337] avg=131072.0000 max=295153 max_rank=5 min=64380 min_rank=0 imbalance_ratio=2.251839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[64380, 163921, 155343, 98292, 74703, 295153, 98447, 98337] avg=131072.0000 max=295153 max_rank=5 min=64380 min_rank=0 imbalance_ratio=2.251839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] avg=131072.0000 max=132383 max_rank=1 min=129898 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=162770 max_load_reduction_pct=0.551477 moved_to_replicas=432671 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[159510, 228357, 130914, 65729, 98408, 65564, 161437, 138657] avg=131072.0000 max=228357 max_rank=1 min=65564 min_rank=5 imbalance_ratio=1.742226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[159510, 228357, 130914, 65729, 98408, 65564, 161437, 138657] avg=131072.0000 max=228357 max_rank=1 min=65564 min_rank=5 imbalance_ratio=1.742226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] avg=131072.0000 max=132383 max_rank=1 min=129898 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=162770 max_load_reduction_pct=0.551477 moved_to_replicas=432671 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[129898, 132383, 132383, 131049, 130422, 132383, 129985, 130073] avg=131072.0000 max=132383 max_rank=1 min=129898 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=162770 max_load_reduction_pct=0.551477 moved_to_replicas=432671 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[97998, 163942, 98325, 98307, 262126, 163709, 164061, 108] avg=131072.0000 max=262126 max_rank=4 min=108 min_rank=7 imbalance_ratio=1.999863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[159510, 228357, 130914, 65729, 98408, 65564, 161437, 138657] avg=131072.0000 max=228357 max_rank=1 min=65564 min_rank=5 imbalance_ratio=1.742226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[97998, 163942, 98325, 98307, 262126, 163709, 164061, 108] avg=131072.0000 max=262126 max_rank=4 min=108 min_rank=7 imbalance_ratio=1.999863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127462 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95974 max_load_reduction_pct=0.420281 moved_to_replicas=317197 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127462 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95974 max_load_reduction_pct=0.420281 moved_to_replicas=317197 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[159510, 228357, 130914, 65729, 98408, 65564, 161437, 138657] avg=131072.0000 max=228357 max_rank=1 min=65564 min_rank=5 imbalance_ratio=1.742226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] avg=131072.0000 max=132383 max_rank=1 min=129466 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129743 max_load_reduction_pct=0.494964 moved_to_replicas=448686 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127462 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95974 max_load_reduction_pct=0.420281 moved_to_replicas=317197 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] avg=131072.0000 max=132383 max_rank=1 min=129466 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129743 max_load_reduction_pct=0.494964 moved_to_replicas=448686 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[159510, 228357, 130914, 65729, 98408, 65564, 161437, 138657] avg=131072.0000 max=228357 max_rank=1 min=65564 min_rank=5 imbalance_ratio=1.742226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[159510, 228357, 130914, 65729, 98408, 65564, 161437, 138657] avg=131072.0000 max=228357 max_rank=1 min=65564 min_rank=5 imbalance_ratio=1.742226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127462 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95974 max_load_reduction_pct=0.420281 moved_to_replicas=317197 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[97998, 163942, 98325, 98307, 262126, 163709, 164061, 108] avg=131072.0000 max=262126 max_rank=4 min=108 min_rank=7 imbalance_ratio=1.999863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[97998, 163942, 98325, 98307, 262126, 163709, 164061, 108] avg=131072.0000 max=262126 max_rank=4 min=108 min_rank=7 imbalance_ratio=1.999863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[98301, 229310, 80471, 115823, 98146, 163846, 98818, 163861] avg=131072.0000 max=229310 max_rank=1 min=80471 min_rank=2 imbalance_ratio=1.749496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127462 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95974 max_load_reduction_pct=0.420281 moved_to_replicas=317197 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[132382, 132383, 130914, 131890, 127462, 128779, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127462 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95974 max_load_reduction_pct=0.420281 moved_to_replicas=317197 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[98301, 229310, 80471, 115823, 98146, 163846, 98818, 163861] avg=131072.0000 max=229310 max_rank=1 min=80471 min_rank=2 imbalance_ratio=1.749496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] avg=131072.0000 max=132383 max_rank=1 min=129466 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129743 max_load_reduction_pct=0.494964 moved_to_replicas=448686 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[97998, 163942, 98325, 98307, 262126, 163709, 164061, 108] avg=131072.0000 max=262126 max_rank=4 min=108 min_rank=7 imbalance_ratio=1.999863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[97998, 163942, 98325, 98307, 262126, 163709, 164061, 108] avg=131072.0000 max=262126 max_rank=4 min=108 min_rank=7 imbalance_ratio=1.999863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] avg=131072.0000 max=132383 max_rank=1 min=129466 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129743 max_load_reduction_pct=0.494964 moved_to_replicas=448686 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] avg=131072.0000 max=132383 max_rank=1 min=128134 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=96927 max_load_reduction_pct=0.422690 moved_to_replicas=319860 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] avg=131072.0000 max=132383 max_rank=1 min=128134 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=96927 max_load_reduction_pct=0.422690 moved_to_replicas=319860 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[97998, 163942, 98325, 98307, 262126, 163709, 164061, 108] avg=131072.0000 max=262126 max_rank=4 min=108 min_rank=7 imbalance_ratio=1.999863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[97998, 163942, 98325, 98307, 262126, 163709, 164061, 108] avg=131072.0000 max=262126 max_rank=4 min=108 min_rank=7 imbalance_ratio=1.999863 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] avg=131072.0000 max=132383 max_rank=1 min=129466 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129743 max_load_reduction_pct=0.494964 moved_to_replicas=448686 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] avg=131072.0000 max=132383 max_rank=1 min=129466 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129743 max_load_reduction_pct=0.494964 moved_to_replicas=448686 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[98301, 229310, 80471, 115823, 98146, 163846, 98818, 163861] avg=131072.0000 max=229310 max_rank=1 min=80471 min_rank=2 imbalance_ratio=1.749496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[98301, 229310, 80471, 115823, 98146, 163846, 98818, 163861] avg=131072.0000 max=229310 max_rank=1 min=80471 min_rank=2 imbalance_ratio=1.749496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[130972, 234123, 130566, 65584, 131786, 130968, 126227, 98350] avg=131072.0000 max=234123 max_rank=1 min=65584 min_rank=3 imbalance_ratio=1.786217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] avg=131072.0000 max=132383 max_rank=1 min=129466 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129743 max_load_reduction_pct=0.494964 moved_to_replicas=448686 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[129466, 132383, 129884, 129985, 132383, 132383, 132383, 129709] avg=131072.0000 max=132383 max_rank=1 min=129466 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129743 max_load_reduction_pct=0.494964 moved_to_replicas=448686 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[130972, 234123, 130566, 65584, 131786, 130968, 126227, 98350] avg=131072.0000 max=234123 max_rank=1 min=65584 min_rank=3 imbalance_ratio=1.786217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] avg=131072.0000 max=132383 max_rank=1 min=128134 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=96927 max_load_reduction_pct=0.422690 moved_to_replicas=319860 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[98301, 229310, 80471, 115823, 98146, 163846, 98818, 163861] avg=131072.0000 max=229310 max_rank=1 min=80471 min_rank=2 imbalance_ratio=1.749496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[98301, 229310, 80471, 115823, 98146, 163846, 98818, 163861] avg=131072.0000 max=229310 max_rank=1 min=80471 min_rank=2 imbalance_ratio=1.749496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] avg=131072.0000 max=132383 max_rank=1 min=128134 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=96927 max_load_reduction_pct=0.422690 moved_to_replicas=319860 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] avg=131072.0000 max=132383 max_rank=1 min=129697 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101740 max_load_reduction_pct=0.434558 moved_to_replicas=204874 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] avg=131072.0000 max=132383 max_rank=1 min=129697 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101740 max_load_reduction_pct=0.434558 moved_to_replicas=204874 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[98301, 229310, 80471, 115823, 98146, 163846, 98818, 163861] avg=131072.0000 max=229310 max_rank=1 min=80471 min_rank=2 imbalance_ratio=1.749496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] avg=131072.0000 max=132383 max_rank=1 min=128134 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=96927 max_load_reduction_pct=0.422690 moved_to_replicas=319860 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[98301, 229310, 80471, 115823, 98146, 163846, 98818, 163861] avg=131072.0000 max=229310 max_rank=1 min=80471 min_rank=2 imbalance_ratio=1.749496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] avg=131072.0000 max=132383 max_rank=1 min=128134 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=96927 max_load_reduction_pct=0.422690 moved_to_replicas=319860 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[130972, 234123, 130566, 65584, 131786, 130968, 126227, 98350] avg=131072.0000 max=234123 max_rank=1 min=65584 min_rank=3 imbalance_ratio=1.786217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[163651, 66850, 65591, 162742, 164010, 130943, 163745, 131044] avg=131072.0000 max=164010 max_rank=4 min=65591 min_rank=2 imbalance_ratio=1.251297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] avg=131072.0000 max=132383 max_rank=1 min=128134 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=96927 max_load_reduction_pct=0.422690 moved_to_replicas=319860 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[130972, 234123, 130566, 65584, 131786, 130968, 126227, 98350] avg=131072.0000 max=234123 max_rank=1 min=65584 min_rank=3 imbalance_ratio=1.786217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[163651, 66850, 65591, 162742, 164010, 130943, 163745, 131044] avg=131072.0000 max=164010 max_rank=4 min=65591 min_rank=2 imbalance_ratio=1.251297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[129711, 132383, 132382, 128134, 130904, 132383, 130296, 132383] avg=131072.0000 max=132383 max_rank=1 min=128134 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=96927 max_load_reduction_pct=0.422690 moved_to_replicas=319860 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] avg=131072.0000 max=132383 max_rank=1 min=129697 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101740 max_load_reduction_pct=0.434558 moved_to_replicas=204874 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[130972, 234123, 130566, 65584, 131786, 130968, 126227, 98350] avg=131072.0000 max=234123 max_rank=1 min=65584 min_rank=3 imbalance_ratio=1.786217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] avg=131072.0000 max=132383 max_rank=0 min=128486 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31627 max_load_reduction_pct=0.192836 moved_to_replicas=249813 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] avg=131072.0000 max=132383 max_rank=1 min=129697 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101740 max_load_reduction_pct=0.434558 moved_to_replicas=204874 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[130972, 234123, 130566, 65584, 131786, 130968, 126227, 98350] avg=131072.0000 max=234123 max_rank=1 min=65584 min_rank=3 imbalance_ratio=1.786217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] avg=131072.0000 max=132383 max_rank=0 min=128486 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31627 max_load_reduction_pct=0.192836 moved_to_replicas=249813 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[130972, 234123, 130566, 65584, 131786, 130968, 126227, 98350] avg=131072.0000 max=234123 max_rank=1 min=65584 min_rank=3 imbalance_ratio=1.786217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] avg=131072.0000 max=132383 max_rank=1 min=129697 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101740 max_load_reduction_pct=0.434558 moved_to_replicas=204874 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[163651, 66850, 65591, 162742, 164010, 130943, 163745, 131044] avg=131072.0000 max=164010 max_rank=4 min=65591 min_rank=2 imbalance_ratio=1.251297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[130972, 234123, 130566, 65584, 131786, 130968, 126227, 98350] avg=131072.0000 max=234123 max_rank=1 min=65584 min_rank=3 imbalance_ratio=1.786217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] avg=131072.0000 max=132383 max_rank=1 min=129697 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101740 max_load_reduction_pct=0.434558 moved_to_replicas=204874 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[127553, 98303, 229259, 133722, 97401, 196588, 65578, 100172] avg=131072.0000 max=229259 max_rank=2 min=65578 min_rank=6 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] avg=131072.0000 max=132383 max_rank=1 min=129697 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101740 max_load_reduction_pct=0.434558 moved_to_replicas=204874 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[163651, 66850, 65591, 162742, 164010, 130943, 163745, 131044] avg=131072.0000 max=164010 max_rank=4 min=65591 min_rank=2 imbalance_ratio=1.251297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[127553, 98303, 229259, 133722, 97401, 196588, 65578, 100172] avg=131072.0000 max=229259 max_rank=2 min=65578 min_rank=6 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] avg=131072.0000 max=132383 max_rank=0 min=128486 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31627 max_load_reduction_pct=0.192836 moved_to_replicas=249813 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[130972, 132383, 130566, 131098, 131786, 130968, 129697, 131106] avg=131072.0000 max=132383 max_rank=1 min=129697 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101740 max_load_reduction_pct=0.434558 moved_to_replicas=204874 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[163651, 66850, 65591, 162742, 164010, 130943, 163745, 131044] avg=131072.0000 max=164010 max_rank=4 min=65591 min_rank=2 imbalance_ratio=1.251297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] avg=131072.0000 max=132383 max_rank=2 min=128892 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=327324 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] avg=131072.0000 max=132383 max_rank=0 min=128486 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31627 max_load_reduction_pct=0.192836 moved_to_replicas=249813 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[163651, 66850, 65591, 162742, 164010, 130943, 163745, 131044] avg=131072.0000 max=164010 max_rank=4 min=65591 min_rank=2 imbalance_ratio=1.251297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] avg=131072.0000 max=132383 max_rank=2 min=128892 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=327324 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[163651, 66850, 65591, 162742, 164010, 130943, 163745, 131044] avg=131072.0000 max=164010 max_rank=4 min=65591 min_rank=2 imbalance_ratio=1.251297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[127553, 98303, 229259, 133722, 97401, 196588, 65578, 100172] avg=131072.0000 max=229259 max_rank=2 min=65578 min_rank=6 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] avg=131072.0000 max=132383 max_rank=0 min=128486 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31627 max_load_reduction_pct=0.192836 moved_to_replicas=249813 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[163651, 66850, 65591, 162742, 164010, 130943, 163745, 131044] avg=131072.0000 max=164010 max_rank=4 min=65591 min_rank=2 imbalance_ratio=1.251297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] avg=131072.0000 max=132383 max_rank=0 min=128486 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31627 max_load_reduction_pct=0.192836 moved_to_replicas=249813 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[229410, 134269, 32810, 65607, 32914, 160728, 229240, 163598] avg=131072.0000 max=229410 max_rank=0 min=32810 min_rank=2 imbalance_ratio=1.750259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] avg=131072.0000 max=132383 max_rank=0 min=128486 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31627 max_load_reduction_pct=0.192836 moved_to_replicas=249813 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[127553, 98303, 229259, 133722, 97401, 196588, 65578, 100172] avg=131072.0000 max=229259 max_rank=2 min=65578 min_rank=6 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[229410, 134269, 32810, 65607, 32914, 160728, 229240, 163598] avg=131072.0000 max=229410 max_rank=0 min=32810 min_rank=2 imbalance_ratio=1.750259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] avg=131072.0000 max=132383 max_rank=2 min=128892 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=327324 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128571, 128486, 132383, 132383, 130943, 132383, 131044] avg=131072.0000 max=132383 max_rank=0 min=128486 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31627 max_load_reduction_pct=0.192836 moved_to_replicas=249813 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] avg=131072.0000 max=132385 max_rank=5 min=128435 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=97025 max_load_reduction_pct=0.422933 moved_to_replicas=510492 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[127553, 98303, 229259, 133722, 97401, 196588, 65578, 100172] avg=131072.0000 max=229259 max_rank=2 min=65578 min_rank=6 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] avg=131072.0000 max=132383 max_rank=2 min=128892 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=327324 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[127553, 98303, 229259, 133722, 97401, 196588, 65578, 100172] avg=131072.0000 max=229259 max_rank=2 min=65578 min_rank=6 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] avg=131072.0000 max=132385 max_rank=5 min=128435 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=97025 max_load_reduction_pct=0.422933 moved_to_replicas=510492 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[127553, 98303, 229259, 133722, 97401, 196588, 65578, 100172] avg=131072.0000 max=229259 max_rank=2 min=65578 min_rank=6 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[229410, 134269, 32810, 65607, 32914, 160728, 229240, 163598] avg=131072.0000 max=229410 max_rank=0 min=32810 min_rank=2 imbalance_ratio=1.750259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[127553, 98303, 229259, 133722, 97401, 196588, 65578, 100172] avg=131072.0000 max=229259 max_rank=2 min=65578 min_rank=6 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] avg=131072.0000 max=132383 max_rank=2 min=128892 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=327324 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] avg=131072.0000 max=132383 max_rank=2 min=128892 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=327324 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131257, 98275, 131068, 65525, 196460, 65532, 196667] avg=131072.0000 max=196667 max_rank=7 min=65525 min_rank=4 imbalance_ratio=1.500450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] avg=131072.0000 max=132383 max_rank=2 min=128892 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=327324 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] avg=131072.0000 max=132385 max_rank=5 min=128435 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=97025 max_load_reduction_pct=0.422933 moved_to_replicas=510492 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[229410, 134269, 32810, 65607, 32914, 160728, 229240, 163598] avg=131072.0000 max=229410 max_rank=0 min=32810 min_rank=2 imbalance_ratio=1.750259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131257, 98275, 131068, 65525, 196460, 65532, 196667] avg=131072.0000 max=196667 max_rank=7 min=65525 min_rank=4 imbalance_ratio=1.500450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[128892, 129670, 132383, 132383, 130154, 132383, 131090, 131621] avg=131072.0000 max=132383 max_rank=2 min=128892 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=327324 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] avg=131072.0000 max=132383 max_rank=0 min=129596 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64284 max_load_reduction_pct=0.326867 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[229410, 134269, 32810, 65607, 32914, 160728, 229240, 163598] avg=131072.0000 max=229410 max_rank=0 min=32810 min_rank=2 imbalance_ratio=1.750259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[229410, 134269, 32810, 65607, 32914, 160728, 229240, 163598] avg=131072.0000 max=229410 max_rank=0 min=32810 min_rank=2 imbalance_ratio=1.750259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] avg=131072.0000 max=132383 max_rank=0 min=129596 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64284 max_load_reduction_pct=0.326867 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] avg=131072.0000 max=132385 max_rank=5 min=128435 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=97025 max_load_reduction_pct=0.422933 moved_to_replicas=510492 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131257, 98275, 131068, 65525, 196460, 65532, 196667] avg=131072.0000 max=196667 max_rank=7 min=65525 min_rank=4 imbalance_ratio=1.500450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[229410, 134269, 32810, 65607, 32914, 160728, 229240, 163598] avg=131072.0000 max=229410 max_rank=0 min=32810 min_rank=2 imbalance_ratio=1.750259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] avg=131072.0000 max=132385 max_rank=5 min=128435 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=97025 max_load_reduction_pct=0.422933 moved_to_replicas=510492 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[229410, 134269, 32810, 65607, 32914, 160728, 229240, 163598] avg=131072.0000 max=229410 max_rank=0 min=32810 min_rank=2 imbalance_ratio=1.750259 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] avg=131072.0000 max=132385 max_rank=5 min=128435 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=97025 max_load_reduction_pct=0.422933 moved_to_replicas=510492 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] avg=131072.0000 max=132383 max_rank=0 min=129596 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64284 max_load_reduction_pct=0.326867 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] avg=131072.0000 max=132385 max_rank=5 min=128435 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=97025 max_load_reduction_pct=0.422933 moved_to_replicas=510492 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131257, 98275, 131068, 65525, 196460, 65532, 196667] avg=131072.0000 max=196667 max_rank=7 min=65525 min_rank=4 imbalance_ratio=1.500450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128435, 129575, 128649, 132385, 132383, 132383] avg=131072.0000 max=132385 max_rank=5 min=128435 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=97025 max_load_reduction_pct=0.422933 moved_to_replicas=510492 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131257, 98275, 131068, 65525, 196460, 65532, 196667] avg=131072.0000 max=196667 max_rank=7 min=65525 min_rank=4 imbalance_ratio=1.500450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131257, 98275, 131068, 65525, 196460, 65532, 196667] avg=131072.0000 max=196667 max_rank=7 min=65525 min_rank=4 imbalance_ratio=1.500450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] avg=131072.0000 max=132383 max_rank=0 min=129596 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64284 max_load_reduction_pct=0.326867 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131257, 98275, 131068, 65525, 196460, 65532, 196667] avg=131072.0000 max=196667 max_rank=7 min=65525 min_rank=4 imbalance_ratio=1.500450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] avg=131072.0000 max=132383 max_rank=0 min=129596 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64284 max_load_reduction_pct=0.326867 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131257, 98275, 131068, 65525, 196460, 65532, 196667] avg=131072.0000 max=196667 max_rank=7 min=65525 min_rank=4 imbalance_ratio=1.500450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] avg=131072.0000 max=132383 max_rank=0 min=129596 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64284 max_load_reduction_pct=0.326867 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] avg=131072.0000 max=132383 max_rank=0 min=129596 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64284 max_load_reduction_pct=0.326867 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131257, 129596, 131068, 129690, 132383, 129816, 132383] avg=131072.0000 max=132383 max_rank=0 min=129596 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64284 max_load_reduction_pct=0.326867 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=4202807 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=355072 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=4188454 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368768 peak_demand=132383 required_capacity=1.010002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=4215570 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=342528 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=4175813 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=381184 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=4186242 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372608 peak_demand=132383 required_capacity=1.010002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=4179496 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=377088 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=4188187 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=371328 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=4217863 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=341504 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:19:10] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 568.3 | throughput per GPU (TFLOP/s/GPU): 586.6 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.016603E+01 | loss scale: 1.0 | grad norm: 17.704 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=143616 expected_recv=257386 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=143616 expected_recv=256032 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=143616 expected_recv=259903 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=143616 expected_recv=257386 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=143616 expected_recv=256032 actual_valid_recv=258176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=260096 slack_slots=27136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=143616 expected_recv=259903 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=258176 slack_slots=29056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264260 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264260 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=143616 expected_recv=258582 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=143616 expected_recv=250481 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=143616 expected_recv=258582 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=143616 expected_recv=250481 actual_valid_recv=253952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=253952 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=143616 expected_recv=261081 expected_overflow=0 exact_recv_per_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=143616 expected_recv=252272 expected_overflow=0 exact_recv_per_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=143616 expected_recv=261081 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=143616 expected_recv=252272 actual_valid_recv=255360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=143616 expected_recv=262328 expected_overflow=0 exact_recv_per_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=255360 slack_slots=31872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=143616 expected_recv=262406 expected_overflow=0 exact_recv_per_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=143616 expected_recv=262328 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=143616 expected_recv=262406 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=265344 slack_slots=21888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=265216 slack_slots=22016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=143616 expected_recv=258628 expected_overflow=0 exact_recv_per_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=143616 expected_recv=256644 expected_overflow=0 exact_recv_per_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=143616 expected_recv=258628 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=143616 expected_recv=256644 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=261120 slack_slots=26112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=259456 slack_slots=27776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=143616 expected_recv=258050 expected_overflow=0 exact_recv_per_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=268288 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=143616 expected_recv=258050 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=268288 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=260480 slack_slots=26752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=143616 expected_recv=258134 expected_overflow=0 exact_recv_per_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=143616 expected_recv=259099 expected_overflow=0 exact_recv_per_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=143616 expected_recv=258134 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=143616 expected_recv=259099 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=143616 expected_recv=262274 expected_overflow=0 exact_recv_per_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=260864 slack_slots=26368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=143616 expected_recv=262274 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=143616 expected_recv=258581 expected_overflow=0 exact_recv_per_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=143616 expected_recv=258581 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=143616 expected_recv=260140 expected_overflow=0 exact_recv_per_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=143616 expected_recv=260140 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=143616 expected_recv=261448 expected_overflow=0 exact_recv_per_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=143616 expected_recv=261448 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=143616 expected_recv=263452 expected_overflow=0 exact_recv_per_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=143616 expected_recv=256165 expected_overflow=0 exact_recv_per_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=143616 expected_recv=261650 expected_overflow=0 exact_recv_per_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=143616 expected_recv=263452 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=143616 expected_recv=256165 actual_valid_recv=258816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=143616 expected_recv=261650 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=143616 expected_recv=262266 expected_overflow=0 exact_recv_per_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=258816 slack_slots=28416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=143616 expected_recv=262266 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=143616 expected_recv=261939 expected_overflow=0 exact_recv_per_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=143616 expected_recv=261939 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=143616 expected_recv=254838 expected_overflow=0 exact_recv_per_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=143616 expected_recv=261107 expected_overflow=0 exact_recv_per_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=143616 expected_recv=254838 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=143616 expected_recv=261107 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=143616 expected_recv=262706 expected_overflow=0 exact_recv_per_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=257792 slack_slots=29440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=143616 expected_recv=262706 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=143616 expected_recv=256165 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=143616 expected_recv=252395 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=143616 expected_recv=256165 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=143616 expected_recv=252395 actual_valid_recv=255104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=255104 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=143616 expected_recv=262060 expected_overflow=0 exact_recv_per_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=143616 expected_recv=260996 expected_overflow=0 exact_recv_per_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=143616 expected_recv=262060 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=143616 expected_recv=260996 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=143616 expected_recv=259105 expected_overflow=0 exact_recv_per_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=143616 expected_recv=259584 expected_overflow=0 exact_recv_per_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=143616 expected_recv=261109 expected_overflow=0 exact_recv_per_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=143616 expected_recv=259105 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=143616 expected_recv=259584 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=143616 expected_recv=261109 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263424 slack_slots=23808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=143616 expected_recv=261796 expected_overflow=0 exact_recv_per_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=143616 expected_recv=262116 expected_overflow=0 exact_recv_per_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=143616 expected_recv=259590 expected_overflow=0 exact_recv_per_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=143616 expected_recv=261796 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=143616 expected_recv=262116 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=143616 expected_recv=259590 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=143616 expected_recv=257231 expected_overflow=0 exact_recv_per_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=143616 expected_recv=262118 expected_overflow=0 exact_recv_per_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=143616 expected_recv=257231 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=143616 expected_recv=262118 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=259968 slack_slots=27264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=143616 expected_recv=262250 expected_overflow=0 exact_recv_per_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=143616 expected_recv=262250 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=264832 slack_slots=22400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=143616 expected_recv=259601 expected_overflow=0 exact_recv_per_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=143616 expected_recv=256735 expected_overflow=0 exact_recv_per_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=143616 expected_recv=259601 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=143616 expected_recv=259502 expected_overflow=0 exact_recv_per_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=143616 expected_recv=256735 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=143616 expected_recv=259502 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=259072 slack_slots=28160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=143616 expected_recv=258238 expected_overflow=0 exact_recv_per_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=143616 expected_recv=258806 expected_overflow=0 exact_recv_per_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=143616 expected_recv=258238 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=143616 expected_recv=258806 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=260992 slack_slots=26240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=143616 expected_recv=260100 expected_overflow=0 exact_recv_per_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=143616 expected_recv=260944 expected_overflow=0 exact_recv_per_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=143616 expected_recv=260100 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=143616 expected_recv=260944 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=143616 expected_recv=259756 expected_overflow=0 exact_recv_per_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=143616 expected_recv=263850 expected_overflow=0 exact_recv_per_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=143616 expected_recv=263850 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=143616 expected_recv=259756 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=143616 expected_recv=262101 expected_overflow=0 exact_recv_per_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=143616 expected_recv=262101 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=143616 expected_recv=259651 expected_overflow=0 exact_recv_per_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=143616 expected_recv=257496 expected_overflow=0 exact_recv_per_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=143616 expected_recv=259651 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=143616 expected_recv=257496 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=143616 expected_recv=254511 expected_overflow=0 exact_recv_per_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=143616 expected_recv=259286 expected_overflow=0 exact_recv_per_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=260096 slack_slots=27136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=143616 expected_recv=254511 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=143616 expected_recv=259286 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=257792 slack_slots=29440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=143616 expected_recv=259525 expected_overflow=0 exact_recv_per_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=143616 expected_recv=259525 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=143616 expected_recv=262118 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=143616 expected_recv=262388 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=143616 expected_recv=262118 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=143616 expected_recv=262388 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=143616 expected_recv=262011 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=143616 expected_recv=259557 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=143616 expected_recv=262011 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=143616 expected_recv=259557 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=143616 expected_recv=259464 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=143616 expected_recv=259464 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=143616 expected_recv=262082 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=143616 expected_recv=259450 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=143616 expected_recv=262082 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=143616 expected_recv=259450 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=143616 expected_recv=256983 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=143616 expected_recv=256983 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[95664, 149601, 148368, 166726, 108264, 147790, 105580, 126583] avg=131072.0000 max=166726 max_rank=3 min=95664 min_rank=0 imbalance_ratio=1.272018 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=259712 slack_slots=27520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=143616 expected_recv=256889 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=143616 expected_recv=256889 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[95664, 149601, 148368, 166726, 108264, 147790, 105580, 126583] avg=131072.0000 max=166726 max_rank=3 min=95664 min_rank=0 imbalance_ratio=1.272018 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[95664, 149601, 148368, 166726, 108264, 147790, 105580, 126583] avg=131072.0000 max=166726 max_rank=3 min=95664 min_rank=0 imbalance_ratio=1.272018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[95664, 149601, 148368, 166726, 108264, 147790, 105580, 126583] avg=131072.0000 max=166726 max_rank=3 min=95664 min_rank=0 imbalance_ratio=1.272018 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] avg=131072.0000 max=132384 max_rank=5 min=126583 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34342 max_load_reduction_pct=0.205979 moved_to_replicas=162395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[95664, 149601, 148368, 166726, 108264, 147790, 105580, 126583] avg=131072.0000 max=166726 max_rank=3 min=95664 min_rank=0 imbalance_ratio=1.272018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] avg=131072.0000 max=132384 max_rank=5 min=126583 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34342 max_load_reduction_pct=0.205979 moved_to_replicas=162395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[95664, 149601, 148368, 166726, 108264, 147790, 105580, 126583] avg=131072.0000 max=166726 max_rank=3 min=95664 min_rank=0 imbalance_ratio=1.272018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] avg=131072.0000 max=132384 max_rank=5 min=126583 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34342 max_load_reduction_pct=0.205979 moved_to_replicas=162395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] avg=131072.0000 max=132384 max_rank=5 min=126583 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34342 max_load_reduction_pct=0.205979 moved_to_replicas=162395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[144666, 193075, 138228, 155200, 130, 88901, 131970, 196406] avg=131072.0000 max=196406 max_rank=7 min=130 min_rank=4 imbalance_ratio=1.498459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[95664, 149601, 148368, 166726, 108264, 147790, 105580, 126583] avg=131072.0000 max=166726 max_rank=3 min=95664 min_rank=0 imbalance_ratio=1.272018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[95664, 149601, 148368, 166726, 108264, 147790, 105580, 126583] avg=131072.0000 max=166726 max_rank=3 min=95664 min_rank=0 imbalance_ratio=1.272018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] avg=131072.0000 max=132384 max_rank=5 min=126583 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34342 max_load_reduction_pct=0.205979 moved_to_replicas=162395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] avg=131072.0000 max=132384 max_rank=5 min=126583 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34342 max_load_reduction_pct=0.205979 moved_to_replicas=162395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[144666, 193075, 138228, 155200, 130, 88901, 131970, 196406] avg=131072.0000 max=196406 max_rank=7 min=130 min_rank=4 imbalance_ratio=1.498459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[144666, 193075, 138228, 155200, 130, 88901, 131970, 196406] avg=131072.0000 max=196406 max_rank=7 min=130 min_rank=4 imbalance_ratio=1.498459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[144666, 193075, 138228, 155200, 130, 88901, 131970, 196406] avg=131072.0000 max=196406 max_rank=7 min=130 min_rank=4 imbalance_ratio=1.498459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] avg=131072.0000 max=132383 max_rank=0 min=125579 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64023 max_load_reduction_pct=0.325973 moved_to_replicas=328635 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] avg=131072.0000 max=132384 max_rank=5 min=126583 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34342 max_load_reduction_pct=0.205979 moved_to_replicas=162395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132044, 132384, 128033, 126583] avg=131072.0000 max=132384 max_rank=5 min=126583 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34342 max_load_reduction_pct=0.205979 moved_to_replicas=162395 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] avg=131072.0000 max=132383 max_rank=0 min=125579 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64023 max_load_reduction_pct=0.325973 moved_to_replicas=328635 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[144666, 193075, 138228, 155200, 130, 88901, 131970, 196406] avg=131072.0000 max=196406 max_rank=7 min=130 min_rank=4 imbalance_ratio=1.498459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] avg=131072.0000 max=132383 max_rank=0 min=125579 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64023 max_load_reduction_pct=0.325973 moved_to_replicas=328635 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[144666, 193075, 138228, 155200, 130, 88901, 131970, 196406] avg=131072.0000 max=196406 max_rank=7 min=130 min_rank=4 imbalance_ratio=1.498459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] avg=131072.0000 max=132383 max_rank=0 min=125579 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64023 max_load_reduction_pct=0.325973 moved_to_replicas=328635 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[226472, 107882, 150156, 91636, 65595, 195592, 98277, 112966] avg=131072.0000 max=226472 max_rank=0 min=65595 min_rank=4 imbalance_ratio=1.727844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] avg=131072.0000 max=132383 max_rank=0 min=125579 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64023 max_load_reduction_pct=0.325973 moved_to_replicas=328635 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[144666, 193075, 138228, 155200, 130, 88901, 131970, 196406] avg=131072.0000 max=196406 max_rank=7 min=130 min_rank=4 imbalance_ratio=1.498459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[144666, 193075, 138228, 155200, 130, 88901, 131970, 196406] avg=131072.0000 max=196406 max_rank=7 min=130 min_rank=4 imbalance_ratio=1.498459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[226472, 107882, 150156, 91636, 65595, 195592, 98277, 112966] avg=131072.0000 max=226472 max_rank=0 min=65595 min_rank=4 imbalance_ratio=1.727844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] avg=131072.0000 max=132383 max_rank=0 min=125579 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64023 max_load_reduction_pct=0.325973 moved_to_replicas=328635 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[226472, 107882, 150156, 91636, 65595, 195592, 98277, 112966] avg=131072.0000 max=226472 max_rank=0 min=65595 min_rank=4 imbalance_ratio=1.727844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] avg=131072.0000 max=132384 max_rank=0 min=126844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=94088 max_load_reduction_pct=0.415451 moved_to_replicas=352059 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[226472, 107882, 150156, 91636, 65595, 195592, 98277, 112966] avg=131072.0000 max=226472 max_rank=0 min=65595 min_rank=4 imbalance_ratio=1.727844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] avg=131072.0000 max=132383 max_rank=0 min=125579 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64023 max_load_reduction_pct=0.325973 moved_to_replicas=328635 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[132383, 132382, 132383, 132382, 125579, 129114, 131970, 132383] avg=131072.0000 max=132383 max_rank=0 min=125579 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64023 max_load_reduction_pct=0.325973 moved_to_replicas=328635 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] avg=131072.0000 max=132384 max_rank=0 min=126844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=94088 max_load_reduction_pct=0.415451 moved_to_replicas=352059 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[226472, 107882, 150156, 91636, 65595, 195592, 98277, 112966] avg=131072.0000 max=226472 max_rank=0 min=65595 min_rank=4 imbalance_ratio=1.727844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] avg=131072.0000 max=132384 max_rank=0 min=126844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=94088 max_load_reduction_pct=0.415451 moved_to_replicas=352059 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] avg=131072.0000 max=132384 max_rank=0 min=126844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=94088 max_load_reduction_pct=0.415451 moved_to_replicas=352059 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[226472, 107882, 150156, 91636, 65595, 195592, 98277, 112966] avg=131072.0000 max=226472 max_rank=0 min=65595 min_rank=4 imbalance_ratio=1.727844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[93064, 98252, 98421, 196223, 162894, 196329, 47930, 155463] avg=131072.0000 max=196329 max_rank=5 min=47930 min_rank=6 imbalance_ratio=1.497871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] avg=131072.0000 max=132384 max_rank=0 min=126844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=94088 max_load_reduction_pct=0.415451 moved_to_replicas=352059 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[226472, 107882, 150156, 91636, 65595, 195592, 98277, 112966] avg=131072.0000 max=226472 max_rank=0 min=65595 min_rank=4 imbalance_ratio=1.727844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[93064, 98252, 98421, 196223, 162894, 196329, 47930, 155463] avg=131072.0000 max=196329 max_rank=5 min=47930 min_rank=6 imbalance_ratio=1.497871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[226472, 107882, 150156, 91636, 65595, 195592, 98277, 112966] avg=131072.0000 max=226472 max_rank=0 min=65595 min_rank=4 imbalance_ratio=1.727844 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[93064, 98252, 98421, 196223, 162894, 196329, 47930, 155463] avg=131072.0000 max=196329 max_rank=5 min=47930 min_rank=6 imbalance_ratio=1.497871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] avg=131072.0000 max=132384 max_rank=0 min=126844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=94088 max_load_reduction_pct=0.415451 moved_to_replicas=352059 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] avg=131072.0000 max=132384 max_rank=7 min=128401 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63945 max_load_reduction_pct=0.325703 moved_to_replicas=366987 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[93064, 98252, 98421, 196223, 162894, 196329, 47930, 155463] avg=131072.0000 max=196329 max_rank=5 min=47930 min_rank=6 imbalance_ratio=1.497871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] avg=131072.0000 max=132384 max_rank=0 min=126844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=94088 max_load_reduction_pct=0.415451 moved_to_replicas=352059 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[93064, 98252, 98421, 196223, 162894, 196329, 47930, 155463] avg=131072.0000 max=196329 max_rank=5 min=47930 min_rank=6 imbalance_ratio=1.497871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] avg=131072.0000 max=132384 max_rank=7 min=128401 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63945 max_load_reduction_pct=0.325703 moved_to_replicas=366987 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] avg=131072.0000 max=132384 max_rank=7 min=128401 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63945 max_load_reduction_pct=0.325703 moved_to_replicas=366987 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[132384, 132384, 132383, 128696, 131119, 132383, 126844, 132383] avg=131072.0000 max=132384 max_rank=0 min=126844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=94088 max_load_reduction_pct=0.415451 moved_to_replicas=352059 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] avg=131072.0000 max=132384 max_rank=7 min=128401 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63945 max_load_reduction_pct=0.325703 moved_to_replicas=366987 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[93064, 98252, 98421, 196223, 162894, 196329, 47930, 155463] avg=131072.0000 max=196329 max_rank=5 min=47930 min_rank=6 imbalance_ratio=1.497871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[65536, 65601, 196566, 197809, 163538, 65579, 131141, 162806] avg=131072.0000 max=197809 max_rank=3 min=65536 min_rank=0 imbalance_ratio=1.509163 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] avg=131072.0000 max=132384 max_rank=7 min=128401 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63945 max_load_reduction_pct=0.325703 moved_to_replicas=366987 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[93064, 98252, 98421, 196223, 162894, 196329, 47930, 155463] avg=131072.0000 max=196329 max_rank=5 min=47930 min_rank=6 imbalance_ratio=1.497871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[65536, 65601, 196566, 197809, 163538, 65579, 131141, 162806] avg=131072.0000 max=197809 max_rank=3 min=65536 min_rank=0 imbalance_ratio=1.509163 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[65536, 65601, 196566, 197809, 163538, 65579, 131141, 162806] avg=131072.0000 max=197809 max_rank=3 min=65536 min_rank=0 imbalance_ratio=1.509163 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] avg=131072.0000 max=132384 max_rank=7 min=128401 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63945 max_load_reduction_pct=0.325703 moved_to_replicas=366987 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] avg=131072.0000 max=132383 max_rank=2 min=128786 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65426 max_load_reduction_pct=0.330753 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[93064, 98252, 98421, 196223, 162894, 196329, 47930, 155463] avg=131072.0000 max=196329 max_rank=5 min=47930 min_rank=6 imbalance_ratio=1.497871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[65536, 65601, 196566, 197809, 163538, 65579, 131141, 162806] avg=131072.0000 max=197809 max_rank=3 min=65536 min_rank=0 imbalance_ratio=1.509163 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] avg=131072.0000 max=132384 max_rank=7 min=128401 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63945 max_load_reduction_pct=0.325703 moved_to_replicas=366987 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[65536, 65601, 196566, 197809, 163538, 65579, 131141, 162806] avg=131072.0000 max=197809 max_rank=3 min=65536 min_rank=0 imbalance_ratio=1.509163 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] avg=131072.0000 max=132383 max_rank=2 min=128786 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65426 max_load_reduction_pct=0.330753 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] avg=131072.0000 max=132383 max_rank=2 min=128786 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65426 max_load_reduction_pct=0.330753 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[128401, 129328, 128932, 132383, 132383, 132383, 132382, 132384] avg=131072.0000 max=132384 max_rank=7 min=128401 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=63945 max_load_reduction_pct=0.325703 moved_to_replicas=366987 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] avg=131072.0000 max=132383 max_rank=2 min=128786 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65426 max_load_reduction_pct=0.330753 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[65536, 65601, 196566, 197809, 163538, 65579, 131141, 162806] avg=131072.0000 max=197809 max_rank=3 min=65536 min_rank=0 imbalance_ratio=1.509163 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[97056, 149853, 225856, 163798, 98734, 99105, 98135, 116039] avg=131072.0000 max=225856 max_rank=2 min=97056 min_rank=0 imbalance_ratio=1.723145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] avg=131072.0000 max=132383 max_rank=2 min=128786 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65426 max_load_reduction_pct=0.330753 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[65536, 65601, 196566, 197809, 163538, 65579, 131141, 162806] avg=131072.0000 max=197809 max_rank=3 min=65536 min_rank=0 imbalance_ratio=1.509163 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[97056, 149853, 225856, 163798, 98734, 99105, 98135, 116039] avg=131072.0000 max=225856 max_rank=2 min=97056 min_rank=0 imbalance_ratio=1.723145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[97056, 149853, 225856, 163798, 98734, 99105, 98135, 116039] avg=131072.0000 max=225856 max_rank=2 min=97056 min_rank=0 imbalance_ratio=1.723145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] avg=131072.0000 max=132383 max_rank=2 min=128786 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65426 max_load_reduction_pct=0.330753 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] avg=131072.0000 max=132383 max_rank=1 min=127811 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=93473 max_load_reduction_pct=0.413861 moved_to_replicas=287135 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[65536, 65601, 196566, 197809, 163538, 65579, 131141, 162806] avg=131072.0000 max=197809 max_rank=3 min=65536 min_rank=0 imbalance_ratio=1.509163 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[97056, 149853, 225856, 163798, 98734, 99105, 98135, 116039] avg=131072.0000 max=225856 max_rank=2 min=97056 min_rank=0 imbalance_ratio=1.723145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] avg=131072.0000 max=132383 max_rank=1 min=127811 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=93473 max_load_reduction_pct=0.413861 moved_to_replicas=287135 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[97056, 149853, 225856, 163798, 98734, 99105, 98135, 116039] avg=131072.0000 max=225856 max_rank=2 min=97056 min_rank=0 imbalance_ratio=1.723145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] avg=131072.0000 max=132383 max_rank=2 min=128786 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65426 max_load_reduction_pct=0.330753 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] avg=131072.0000 max=132383 max_rank=1 min=127811 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=93473 max_load_reduction_pct=0.413861 moved_to_replicas=287135 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[129453, 128786, 132383, 132383, 132383, 129664, 131141, 132383] avg=131072.0000 max=132383 max_rank=2 min=128786 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65426 max_load_reduction_pct=0.330753 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] avg=131072.0000 max=132383 max_rank=1 min=127811 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=93473 max_load_reduction_pct=0.413861 moved_to_replicas=287135 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] avg=131072.0000 max=132383 max_rank=1 min=127811 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=93473 max_load_reduction_pct=0.413861 moved_to_replicas=287135 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[133017, 65597, 174339, 130986, 184457, 131291, 98262, 130627] avg=131072.0000 max=184457 max_rank=4 min=65597 min_rank=1 imbalance_ratio=1.407295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[97056, 149853, 225856, 163798, 98734, 99105, 98135, 116039] avg=131072.0000 max=225856 max_rank=2 min=97056 min_rank=0 imbalance_ratio=1.723145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[133017, 65597, 174339, 130986, 184457, 131291, 98262, 130627] avg=131072.0000 max=184457 max_rank=4 min=65597 min_rank=1 imbalance_ratio=1.407295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[97056, 149853, 225856, 163798, 98734, 99105, 98135, 116039] avg=131072.0000 max=225856 max_rank=2 min=97056 min_rank=0 imbalance_ratio=1.723145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[133017, 65597, 174339, 130986, 184457, 131291, 98262, 130627] avg=131072.0000 max=184457 max_rank=4 min=65597 min_rank=1 imbalance_ratio=1.407295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[97056, 149853, 225856, 163798, 98734, 99105, 98135, 116039] avg=131072.0000 max=225856 max_rank=2 min=97056 min_rank=0 imbalance_ratio=1.723145 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[133017, 65597, 174339, 130986, 184457, 131291, 98262, 130627] avg=131072.0000 max=184457 max_rank=4 min=65597 min_rank=1 imbalance_ratio=1.407295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] avg=131072.0000 max=132383 max_rank=1 min=127811 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=93473 max_load_reduction_pct=0.413861 moved_to_replicas=287135 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] avg=131072.0000 max=132383 max_rank=0 min=127409 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=52074 max_load_reduction_pct=0.282310 moved_to_replicas=189446 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] avg=131072.0000 max=132383 max_rank=1 min=127811 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=93473 max_load_reduction_pct=0.413861 moved_to_replicas=287135 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[133017, 65597, 174339, 130986, 184457, 131291, 98262, 130627] avg=131072.0000 max=184457 max_rank=4 min=65597 min_rank=1 imbalance_ratio=1.407295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] avg=131072.0000 max=132383 max_rank=0 min=127409 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=52074 max_load_reduction_pct=0.282310 moved_to_replicas=189446 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[129818, 132383, 132381, 132383, 127811, 130520, 130897, 132383] avg=131072.0000 max=132383 max_rank=1 min=127811 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=93473 max_load_reduction_pct=0.413861 moved_to_replicas=287135 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] avg=131072.0000 max=132383 max_rank=0 min=127409 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=52074 max_load_reduction_pct=0.282310 moved_to_replicas=189446 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] avg=131072.0000 max=132383 max_rank=0 min=127409 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=52074 max_load_reduction_pct=0.282310 moved_to_replicas=189446 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[133017, 65597, 174339, 130986, 184457, 131291, 98262, 130627] avg=131072.0000 max=184457 max_rank=4 min=65597 min_rank=1 imbalance_ratio=1.407295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[36167, 204725, 163754, 134491, 150054, 228296, 32808, 98281] avg=131072.0000 max=228296 max_rank=5 min=32808 min_rank=6 imbalance_ratio=1.741760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[133017, 65597, 174339, 130986, 184457, 131291, 98262, 130627] avg=131072.0000 max=184457 max_rank=4 min=65597 min_rank=1 imbalance_ratio=1.407295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[36167, 204725, 163754, 134491, 150054, 228296, 32808, 98281] avg=131072.0000 max=228296 max_rank=5 min=32808 min_rank=6 imbalance_ratio=1.741760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] avg=131072.0000 max=132383 max_rank=0 min=127409 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=52074 max_load_reduction_pct=0.282310 moved_to_replicas=189446 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[133017, 65597, 174339, 130986, 184457, 131291, 98262, 130627] avg=131072.0000 max=184457 max_rank=4 min=65597 min_rank=1 imbalance_ratio=1.407295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[36167, 204725, 163754, 134491, 150054, 228296, 32808, 98281] avg=131072.0000 max=228296 max_rank=5 min=32808 min_rank=6 imbalance_ratio=1.741760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] avg=131072.0000 max=132383 max_rank=0 min=127409 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=52074 max_load_reduction_pct=0.282310 moved_to_replicas=189446 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] avg=131072.0000 max=132383 max_rank=0 min=125559 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=95913 max_load_reduction_pct=0.420126 moved_to_replicas=439170 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[36167, 204725, 163754, 134491, 150054, 228296, 32808, 98281] avg=131072.0000 max=228296 max_rank=5 min=32808 min_rank=6 imbalance_ratio=1.741760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] avg=131072.0000 max=132383 max_rank=0 min=127409 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=52074 max_load_reduction_pct=0.282310 moved_to_replicas=189446 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] avg=131072.0000 max=132383 max_rank=0 min=125559 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=95913 max_load_reduction_pct=0.420126 moved_to_replicas=439170 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131116, 132382, 130986, 132382, 131291, 127409, 130627] avg=131072.0000 max=132383 max_rank=0 min=127409 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=52074 max_load_reduction_pct=0.282310 moved_to_replicas=189446 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] avg=131072.0000 max=132383 max_rank=0 min=125559 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=95913 max_load_reduction_pct=0.420126 moved_to_replicas=439170 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] avg=131072.0000 max=132383 max_rank=0 min=125559 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=95913 max_load_reduction_pct=0.420126 moved_to_replicas=439170 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165321, 32900, 131045, 129431, 32805, 262041, 196724] avg=131072.0000 max=262041 max_rank=6 min=32805 min_rank=5 imbalance_ratio=1.999214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[36167, 204725, 163754, 134491, 150054, 228296, 32808, 98281] avg=131072.0000 max=228296 max_rank=5 min=32808 min_rank=6 imbalance_ratio=1.741760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[36167, 204725, 163754, 134491, 150054, 228296, 32808, 98281] avg=131072.0000 max=228296 max_rank=5 min=32808 min_rank=6 imbalance_ratio=1.741760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165321, 32900, 131045, 129431, 32805, 262041, 196724] avg=131072.0000 max=262041 max_rank=6 min=32805 min_rank=5 imbalance_ratio=1.999214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[36167, 204725, 163754, 134491, 150054, 228296, 32808, 98281] avg=131072.0000 max=228296 max_rank=5 min=32808 min_rank=6 imbalance_ratio=1.741760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165321, 32900, 131045, 129431, 32805, 262041, 196724] avg=131072.0000 max=262041 max_rank=6 min=32805 min_rank=5 imbalance_ratio=1.999214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129608 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129658 max_load_reduction_pct=0.494800 moved_to_replicas=455112 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] avg=131072.0000 max=132383 max_rank=0 min=125559 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=95913 max_load_reduction_pct=0.420126 moved_to_replicas=439170 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165321, 32900, 131045, 129431, 32805, 262041, 196724] avg=131072.0000 max=262041 max_rank=6 min=32805 min_rank=5 imbalance_ratio=1.999214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[36167, 204725, 163754, 134491, 150054, 228296, 32808, 98281] avg=131072.0000 max=228296 max_rank=5 min=32808 min_rank=6 imbalance_ratio=1.741760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] avg=131072.0000 max=132383 max_rank=0 min=125559 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=95913 max_load_reduction_pct=0.420126 moved_to_replicas=439170 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129608 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129658 max_load_reduction_pct=0.494800 moved_to_replicas=455112 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] avg=131072.0000 max=132383 max_rank=0 min=125559 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=95913 max_load_reduction_pct=0.420126 moved_to_replicas=439170 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129608 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129658 max_load_reduction_pct=0.494800 moved_to_replicas=455112 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129608 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129658 max_load_reduction_pct=0.494800 moved_to_replicas=455112 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[131063, 180731, 131072, 65383, 196553, 148131, 98546, 97097] avg=131072.0000 max=196553 max_rank=4 min=65383 min_rank=3 imbalance_ratio=1.499580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 132381, 132383, 128721, 125559] avg=131072.0000 max=132383 max_rank=0 min=125559 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=95913 max_load_reduction_pct=0.420126 moved_to_replicas=439170 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165321, 32900, 131045, 129431, 32805, 262041, 196724] avg=131072.0000 max=262041 max_rank=6 min=32805 min_rank=5 imbalance_ratio=1.999214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[131063, 180731, 131072, 65383, 196553, 148131, 98546, 97097] avg=131072.0000 max=196553 max_rank=4 min=65383 min_rank=3 imbalance_ratio=1.499580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165321, 32900, 131045, 129431, 32805, 262041, 196724] avg=131072.0000 max=262041 max_rank=6 min=32805 min_rank=5 imbalance_ratio=1.999214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165321, 32900, 131045, 129431, 32805, 262041, 196724] avg=131072.0000 max=262041 max_rank=6 min=32805 min_rank=5 imbalance_ratio=1.999214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] avg=131072.0000 max=132385 max_rank=5 min=128507 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64168 max_load_reduction_pct=0.326467 moved_to_replicas=256299 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[131063, 180731, 131072, 65383, 196553, 148131, 98546, 97097] avg=131072.0000 max=196553 max_rank=4 min=65383 min_rank=3 imbalance_ratio=1.499580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129608 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129658 max_load_reduction_pct=0.494800 moved_to_replicas=455112 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[131063, 180731, 131072, 65383, 196553, 148131, 98546, 97097] avg=131072.0000 max=196553 max_rank=4 min=65383 min_rank=3 imbalance_ratio=1.499580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] avg=131072.0000 max=132385 max_rank=5 min=128507 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64168 max_load_reduction_pct=0.326467 moved_to_replicas=256299 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129608 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129658 max_load_reduction_pct=0.494800 moved_to_replicas=455112 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165321, 32900, 131045, 129431, 32805, 262041, 196724] avg=131072.0000 max=262041 max_rank=6 min=32805 min_rank=5 imbalance_ratio=1.999214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129608 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129658 max_load_reduction_pct=0.494800 moved_to_replicas=455112 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] avg=131072.0000 max=132385 max_rank=5 min=128507 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64168 max_load_reduction_pct=0.326467 moved_to_replicas=256299 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] avg=131072.0000 max=132385 max_rank=5 min=128507 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64168 max_load_reduction_pct=0.326467 moved_to_replicas=256299 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[98371, 196475, 97266, 196562, 203880, 88380, 44726, 122916] avg=131072.0000 max=203880 max_rank=4 min=44726 min_rank=6 imbalance_ratio=1.555481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[129889, 132383, 129799, 131045, 129608, 131086, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129608 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129658 max_load_reduction_pct=0.494800 moved_to_replicas=455112 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[131063, 180731, 131072, 65383, 196553, 148131, 98546, 97097] avg=131072.0000 max=196553 max_rank=4 min=65383 min_rank=3 imbalance_ratio=1.499580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[98371, 196475, 97266, 196562, 203880, 88380, 44726, 122916] avg=131072.0000 max=203880 max_rank=4 min=44726 min_rank=6 imbalance_ratio=1.555481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[131063, 180731, 131072, 65383, 196553, 148131, 98546, 97097] avg=131072.0000 max=196553 max_rank=4 min=65383 min_rank=3 imbalance_ratio=1.499580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] avg=131072.0000 max=132383 max_rank=1 min=127120 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=71497 max_load_reduction_pct=0.350682 moved_to_replicas=396490 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[98371, 196475, 97266, 196562, 203880, 88380, 44726, 122916] avg=131072.0000 max=203880 max_rank=4 min=44726 min_rank=6 imbalance_ratio=1.555481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[131063, 180731, 131072, 65383, 196553, 148131, 98546, 97097] avg=131072.0000 max=196553 max_rank=4 min=65383 min_rank=3 imbalance_ratio=1.499580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[98371, 196475, 97266, 196562, 203880, 88380, 44726, 122916] avg=131072.0000 max=203880 max_rank=4 min=44726 min_rank=6 imbalance_ratio=1.555481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] avg=131072.0000 max=132385 max_rank=5 min=128507 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64168 max_load_reduction_pct=0.326467 moved_to_replicas=256299 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] avg=131072.0000 max=132383 max_rank=1 min=127120 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=71497 max_load_reduction_pct=0.350682 moved_to_replicas=396490 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] avg=131072.0000 max=132385 max_rank=5 min=128507 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64168 max_load_reduction_pct=0.326467 moved_to_replicas=256299 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[131063, 180731, 131072, 65383, 196553, 148131, 98546, 97097] avg=131072.0000 max=196553 max_rank=4 min=65383 min_rank=3 imbalance_ratio=1.499580 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] avg=131072.0000 max=132383 max_rank=1 min=127120 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=71497 max_load_reduction_pct=0.350682 moved_to_replicas=396490 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] avg=131072.0000 max=132385 max_rank=5 min=128507 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64168 max_load_reduction_pct=0.326467 moved_to_replicas=256299 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] avg=131072.0000 max=132383 max_rank=1 min=127120 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=71497 max_load_reduction_pct=0.350682 moved_to_replicas=396490 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[97031, 229367, 65254, 65687, 229411, 163894, 165109, 32823] avg=131072.0000 max=229411 max_rank=4 min=32823 min_rank=7 imbalance_ratio=1.750267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[131063, 132382, 131072, 130903, 132383, 132385, 129881, 128507] avg=131072.0000 max=132385 max_rank=5 min=128507 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64168 max_load_reduction_pct=0.326467 moved_to_replicas=256299 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[98371, 196475, 97266, 196562, 203880, 88380, 44726, 122916] avg=131072.0000 max=203880 max_rank=4 min=44726 min_rank=6 imbalance_ratio=1.555481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[97031, 229367, 65254, 65687, 229411, 163894, 165109, 32823] avg=131072.0000 max=229411 max_rank=4 min=32823 min_rank=7 imbalance_ratio=1.750267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[98371, 196475, 97266, 196562, 203880, 88380, 44726, 122916] avg=131072.0000 max=203880 max_rank=4 min=44726 min_rank=6 imbalance_ratio=1.555481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] avg=131072.0000 max=132383 max_rank=1 min=128557 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97028 max_load_reduction_pct=0.422944 moved_to_replicas=518258 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[97031, 229367, 65254, 65687, 229411, 163894, 165109, 32823] avg=131072.0000 max=229411 max_rank=4 min=32823 min_rank=7 imbalance_ratio=1.750267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[97031, 229367, 65254, 65687, 229411, 163894, 165109, 32823] avg=131072.0000 max=229411 max_rank=4 min=32823 min_rank=7 imbalance_ratio=1.750267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[98371, 196475, 97266, 196562, 203880, 88380, 44726, 122916] avg=131072.0000 max=203880 max_rank=4 min=44726 min_rank=6 imbalance_ratio=1.555481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] avg=131072.0000 max=132383 max_rank=1 min=127120 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=71497 max_load_reduction_pct=0.350682 moved_to_replicas=396490 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] avg=131072.0000 max=132383 max_rank=1 min=128557 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97028 max_load_reduction_pct=0.422944 moved_to_replicas=518258 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[98371, 196475, 97266, 196562, 203880, 88380, 44726, 122916] avg=131072.0000 max=203880 max_rank=4 min=44726 min_rank=6 imbalance_ratio=1.555481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] avg=131072.0000 max=132383 max_rank=1 min=127120 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=71497 max_load_reduction_pct=0.350682 moved_to_replicas=396490 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] avg=131072.0000 max=132383 max_rank=1 min=128557 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97028 max_load_reduction_pct=0.422944 moved_to_replicas=518258 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] avg=131072.0000 max=132383 max_rank=1 min=128557 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97028 max_load_reduction_pct=0.422944 moved_to_replicas=518258 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] avg=131072.0000 max=132383 max_rank=1 min=127120 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=71497 max_load_reduction_pct=0.350682 moved_to_replicas=396490 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[131471, 163893, 98369, 98331, 196570, 98629, 163022, 98291] avg=131072.0000 max=196570 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[131131, 132383, 128685, 132383, 132382, 127120, 132383, 132109] avg=131072.0000 max=132383 max_rank=1 min=127120 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=71497 max_load_reduction_pct=0.350682 moved_to_replicas=396490 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[131471, 163893, 98369, 98331, 196570, 98629, 163022, 98291] avg=131072.0000 max=196570 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[97031, 229367, 65254, 65687, 229411, 163894, 165109, 32823] avg=131072.0000 max=229411 max_rank=4 min=32823 min_rank=7 imbalance_ratio=1.750267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[97031, 229367, 65254, 65687, 229411, 163894, 165109, 32823] avg=131072.0000 max=229411 max_rank=4 min=32823 min_rank=7 imbalance_ratio=1.750267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129268 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64187 max_load_reduction_pct=0.326535 moved_to_replicas=251374 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[131471, 163893, 98369, 98331, 196570, 98629, 163022, 98291] avg=131072.0000 max=196570 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[131471, 163893, 98369, 98331, 196570, 98629, 163022, 98291] avg=131072.0000 max=196570 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129268 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64187 max_load_reduction_pct=0.326535 moved_to_replicas=251374 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[97031, 229367, 65254, 65687, 229411, 163894, 165109, 32823] avg=131072.0000 max=229411 max_rank=4 min=32823 min_rank=7 imbalance_ratio=1.750267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[97031, 229367, 65254, 65687, 229411, 163894, 165109, 32823] avg=131072.0000 max=229411 max_rank=4 min=32823 min_rank=7 imbalance_ratio=1.750267 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] avg=131072.0000 max=132383 max_rank=1 min=128557 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97028 max_load_reduction_pct=0.422944 moved_to_replicas=518258 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] avg=131072.0000 max=132383 max_rank=1 min=128557 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97028 max_load_reduction_pct=0.422944 moved_to_replicas=518258 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129268 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64187 max_load_reduction_pct=0.326535 moved_to_replicas=251374 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129268 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64187 max_load_reduction_pct=0.326535 moved_to_replicas=251374 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] avg=131072.0000 max=132383 max_rank=1 min=128557 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97028 max_load_reduction_pct=0.422944 moved_to_replicas=518258 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[96859, 100906, 163946, 194228, 65636, 166269, 65627, 195105] avg=131072.0000 max=195105 max_rank=7 min=65627 min_rank=6 imbalance_ratio=1.488533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[129789, 132383, 130740, 129958, 132383, 132383, 132383, 128557] avg=131072.0000 max=132383 max_rank=1 min=128557 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97028 max_load_reduction_pct=0.422944 moved_to_replicas=518258 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[96859, 100906, 163946, 194228, 65636, 166269, 65627, 195105] avg=131072.0000 max=195105 max_rank=7 min=65627 min_rank=6 imbalance_ratio=1.488533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[131471, 163893, 98369, 98331, 196570, 98629, 163022, 98291] avg=131072.0000 max=196570 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[131471, 163893, 98369, 98331, 196570, 98629, 163022, 98291] avg=131072.0000 max=196570 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[96859, 100906, 163946, 194228, 65636, 166269, 65627, 195105] avg=131072.0000 max=195105 max_rank=7 min=65627 min_rank=6 imbalance_ratio=1.488533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[96859, 100906, 163946, 194228, 65636, 166269, 65627, 195105] avg=131072.0000 max=195105 max_rank=7 min=65627 min_rank=6 imbalance_ratio=1.488533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] avg=131072.0000 max=132383 max_rank=1 min=127160 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62722 max_load_reduction_pct=0.321478 moved_to_replicas=380474 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] avg=131072.0000 max=132383 max_rank=1 min=127160 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62722 max_load_reduction_pct=0.321478 moved_to_replicas=380474 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129268 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64187 max_load_reduction_pct=0.326535 moved_to_replicas=251374 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129268 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64187 max_load_reduction_pct=0.326535 moved_to_replicas=251374 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[131471, 163893, 98369, 98331, 196570, 98629, 163022, 98291] avg=131072.0000 max=196570 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[131471, 163893, 98369, 98331, 196570, 98629, 163022, 98291] avg=131072.0000 max=196570 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] avg=131072.0000 max=132383 max_rank=1 min=127160 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62722 max_load_reduction_pct=0.321478 moved_to_replicas=380474 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] avg=131072.0000 max=132383 max_rank=1 min=127160 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62722 max_load_reduction_pct=0.321478 moved_to_replicas=380474 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[229298, 196620, 72, 131067, 98300, 131078, 131116, 131025] avg=131072.0000 max=229298 max_rank=0 min=72 min_rank=2 imbalance_ratio=1.749405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[229298, 196620, 72, 131067, 98300, 131078, 131116, 131025] avg=131072.0000 max=229298 max_rank=0 min=72 min_rank=2 imbalance_ratio=1.749405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129268 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64187 max_load_reduction_pct=0.326535 moved_to_replicas=251374 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[131471, 132383, 129879, 129758, 132383, 129268, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129268 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64187 max_load_reduction_pct=0.326535 moved_to_replicas=251374 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[96859, 100906, 163946, 194228, 65636, 166269, 65627, 195105] avg=131072.0000 max=195105 max_rank=7 min=65627 min_rank=6 imbalance_ratio=1.488533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[96859, 100906, 163946, 194228, 65636, 166269, 65627, 195105] avg=131072.0000 max=195105 max_rank=7 min=65627 min_rank=6 imbalance_ratio=1.488533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[229298, 196620, 72, 131067, 98300, 131078, 131116, 131025] avg=131072.0000 max=229298 max_rank=0 min=72 min_rank=2 imbalance_ratio=1.749405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[229298, 196620, 72, 131067, 98300, 131078, 131116, 131025] avg=131072.0000 max=229298 max_rank=0 min=72 min_rank=2 imbalance_ratio=1.749405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96915 max_load_reduction_pct=0.422660 moved_to_replicas=322285 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96915 max_load_reduction_pct=0.422660 moved_to_replicas=322285 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] avg=131072.0000 max=132383 max_rank=1 min=127160 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62722 max_load_reduction_pct=0.321478 moved_to_replicas=380474 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] avg=131072.0000 max=132383 max_rank=1 min=127160 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62722 max_load_reduction_pct=0.321478 moved_to_replicas=380474 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[96859, 100906, 163946, 194228, 65636, 166269, 65627, 195105] avg=131072.0000 max=195105 max_rank=7 min=65627 min_rank=6 imbalance_ratio=1.488533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[96859, 100906, 163946, 194228, 65636, 166269, 65627, 195105] avg=131072.0000 max=195105 max_rank=7 min=65627 min_rank=6 imbalance_ratio=1.488533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96915 max_load_reduction_pct=0.422660 moved_to_replicas=322285 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96915 max_load_reduction_pct=0.422660 moved_to_replicas=322285 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] avg=131072.0000 max=132383 max_rank=1 min=127160 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62722 max_load_reduction_pct=0.321478 moved_to_replicas=380474 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[196364, 196611, 32839, 163757, 32820, 163771, 65593, 196821] avg=131072.0000 max=196821 max_rank=7 min=32820 min_rank=4 imbalance_ratio=1.501625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[196364, 196611, 32839, 163757, 32820, 163771, 65593, 196821] avg=131072.0000 max=196821 max_rank=7 min=32820 min_rank=4 imbalance_ratio=1.501625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[127160, 132383, 132383, 132383, 128357, 132383, 131144, 132383] avg=131072.0000 max=132383 max_rank=1 min=127160 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62722 max_load_reduction_pct=0.321478 moved_to_replicas=380474 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[229298, 196620, 72, 131067, 98300, 131078, 131116, 131025] avg=131072.0000 max=229298 max_rank=0 min=72 min_rank=2 imbalance_ratio=1.749405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[229298, 196620, 72, 131067, 98300, 131078, 131116, 131025] avg=131072.0000 max=229298 max_rank=0 min=72 min_rank=2 imbalance_ratio=1.749405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[196364, 196611, 32839, 163757, 32820, 163771, 65593, 196821] avg=131072.0000 max=196821 max_rank=7 min=32820 min_rank=4 imbalance_ratio=1.501625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[196364, 196611, 32839, 163757, 32820, 163771, 65593, 196821] avg=131072.0000 max=196821 max_rank=7 min=32820 min_rank=4 imbalance_ratio=1.501625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] avg=131072.0000 max=132383 max_rank=0 min=128271 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64438 max_load_reduction_pct=0.327394 moved_to_replicas=510662 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] avg=131072.0000 max=132383 max_rank=0 min=128271 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64438 max_load_reduction_pct=0.327394 moved_to_replicas=510662 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96915 max_load_reduction_pct=0.422660 moved_to_replicas=322285 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96915 max_load_reduction_pct=0.422660 moved_to_replicas=322285 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[229298, 196620, 72, 131067, 98300, 131078, 131116, 131025] avg=131072.0000 max=229298 max_rank=0 min=72 min_rank=2 imbalance_ratio=1.749405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[229298, 196620, 72, 131067, 98300, 131078, 131116, 131025] avg=131072.0000 max=229298 max_rank=0 min=72 min_rank=2 imbalance_ratio=1.749405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] avg=131072.0000 max=132383 max_rank=0 min=128271 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64438 max_load_reduction_pct=0.327394 moved_to_replicas=510662 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] avg=131072.0000 max=132383 max_rank=0 min=128271 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64438 max_load_reduction_pct=0.327394 moved_to_replicas=510662 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96915 max_load_reduction_pct=0.422660 moved_to_replicas=322285 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131067, 129777, 131078, 131116, 131025] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96915 max_load_reduction_pct=0.422660 moved_to_replicas=322285 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[196364, 196611, 32839, 163757, 32820, 163771, 65593, 196821] avg=131072.0000 max=196821 max_rank=7 min=32820 min_rank=4 imbalance_ratio=1.501625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[196364, 196611, 32839, 163757, 32820, 163771, 65593, 196821] avg=131072.0000 max=196821 max_rank=7 min=32820 min_rank=4 imbalance_ratio=1.501625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] avg=131072.0000 max=132383 max_rank=0 min=128271 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64438 max_load_reduction_pct=0.327394 moved_to_replicas=510662 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[196364, 196611, 32839, 163757, 32820, 163771, 65593, 196821] avg=131072.0000 max=196821 max_rank=7 min=32820 min_rank=4 imbalance_ratio=1.501625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] avg=131072.0000 max=132383 max_rank=0 min=128271 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64438 max_load_reduction_pct=0.327394 moved_to_replicas=510662 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[196364, 196611, 32839, 163757, 32820, 163771, 65593, 196821] avg=131072.0000 max=196821 max_rank=7 min=32820 min_rank=4 imbalance_ratio=1.501625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] avg=131072.0000 max=132383 max_rank=0 min=128271 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64438 max_load_reduction_pct=0.327394 moved_to_replicas=510662 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128664, 132383, 128271, 132383, 129726, 132383] avg=131072.0000 max=132383 max_rank=0 min=128271 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64438 max_load_reduction_pct=0.327394 moved_to_replicas=510662 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=4220988 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=336384 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=4173187 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=381312 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=4184955 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=373376 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=4191775 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364800 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=4182249 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=372992 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=4191555 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364800 peak_demand=132383 required_capacity=1.010002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=4209883 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=346752 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=4199840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=358784 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:19:10] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 563.9 | throughput per GPU (TFLOP/s/GPU): 591.2 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.901007E+00 | loss scale: 1.0 | grad norm: 6.199 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=143616 expected_recv=254552 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=143616 expected_recv=258193 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=143616 expected_recv=262837 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=143616 expected_recv=262507 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=143616 expected_recv=254552 actual_valid_recv=257152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=143616 expected_recv=258193 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=143616 expected_recv=262837 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=257152 slack_slots=30080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=143616 expected_recv=262507 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=264960 slack_slots=22272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=143616 expected_recv=260851 expected_overflow=0 exact_recv_per_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=143616 expected_recv=261986 expected_overflow=0 exact_recv_per_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=143616 expected_recv=255754 expected_overflow=0 exact_recv_per_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=143616 expected_recv=260851 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=143616 expected_recv=261986 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=268032 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=143616 expected_recv=255754 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=143616 expected_recv=259497 expected_overflow=0 exact_recv_per_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=258688 slack_slots=28544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=143616 expected_recv=259497 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=143616 expected_recv=249942 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=143616 expected_recv=249942 actual_valid_recv=252416 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=143616 expected_recv=263361 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=143616 expected_recv=261680 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=252416 slack_slots=34816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=143616 expected_recv=263107 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=143616 expected_recv=263361 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=143616 expected_recv=261680 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=143616 expected_recv=263107 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=265472 slack_slots=21760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=143616 expected_recv=255191 expected_overflow=0 exact_recv_per_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=143616 expected_recv=262047 expected_overflow=0 exact_recv_per_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=143616 expected_recv=255191 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=143616 expected_recv=258827 expected_overflow=0 exact_recv_per_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=143616 expected_recv=262047 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=257408 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=143616 expected_recv=262024 expected_overflow=0 exact_recv_per_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=143616 expected_recv=258827 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=143616 expected_recv=262024 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=143616 expected_recv=262076 expected_overflow=0 exact_recv_per_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=143616 expected_recv=262076 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=143616 expected_recv=255015 expected_overflow=0 exact_recv_per_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=143616 expected_recv=255015 actual_valid_recv=257536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=143616 expected_recv=259808 expected_overflow=0 exact_recv_per_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=143616 expected_recv=261190 expected_overflow=0 exact_recv_per_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=257536 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=143616 expected_recv=259808 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=143616 expected_recv=261190 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=143616 expected_recv=262070 expected_overflow=0 exact_recv_per_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=143616 expected_recv=262070 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=143616 expected_recv=252750 expected_overflow=0 exact_recv_per_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=143616 expected_recv=261983 expected_overflow=0 exact_recv_per_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=143616 expected_recv=252750 actual_valid_recv=255104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=143616 expected_recv=261995 expected_overflow=0 exact_recv_per_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=143616 expected_recv=264057 expected_overflow=0 exact_recv_per_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=143616 expected_recv=261983 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=255104 slack_slots=32128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=143616 expected_recv=261995 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=143616 expected_recv=259228 expected_overflow=0 exact_recv_per_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=143616 expected_recv=264057 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=143616 expected_recv=259228 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=143616 expected_recv=262550 expected_overflow=0 exact_recv_per_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=143616 expected_recv=262550 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=143616 expected_recv=260746 expected_overflow=0 exact_recv_per_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=143616 expected_recv=262167 expected_overflow=0 exact_recv_per_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=143616 expected_recv=258162 expected_overflow=0 exact_recv_per_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=143616 expected_recv=260746 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=143616 expected_recv=258162 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=143616 expected_recv=262167 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=143616 expected_recv=259618 expected_overflow=0 exact_recv_per_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=260480 slack_slots=26752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=143616 expected_recv=262152 expected_overflow=0 exact_recv_per_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=143616 expected_recv=259618 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=143616 expected_recv=262152 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=143616 expected_recv=257225 expected_overflow=0 exact_recv_per_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=143616 expected_recv=257225 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=143616 expected_recv=259093 expected_overflow=0 exact_recv_per_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=259328 slack_slots=27904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=143616 expected_recv=259093 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=143616 expected_recv=261835 expected_overflow=0 exact_recv_per_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=143616 expected_recv=261835 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=143616 expected_recv=260466 expected_overflow=0 exact_recv_per_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=143616 expected_recv=260466 actual_valid_recv=262656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=143616 expected_recv=258812 expected_overflow=0 exact_recv_per_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=262656 slack_slots=24576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=143616 expected_recv=259590 expected_overflow=0 exact_recv_per_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=143616 expected_recv=262114 expected_overflow=0 exact_recv_per_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=143616 expected_recv=263570 expected_overflow=0 exact_recv_per_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=143616 expected_recv=262153 expected_overflow=0 exact_recv_per_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=143616 expected_recv=258812 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=143616 expected_recv=259590 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=143616 expected_recv=262114 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=143616 expected_recv=263570 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=143616 expected_recv=262153 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=265856 slack_slots=21376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=143616 expected_recv=262358 expected_overflow=0 exact_recv_per_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=143616 expected_recv=262358 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=143616 expected_recv=255460 expected_overflow=0 exact_recv_per_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=143616 expected_recv=259352 expected_overflow=0 exact_recv_per_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=143616 expected_recv=259352 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=143616 expected_recv=255460 actual_valid_recv=257536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=260864 slack_slots=26368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=257536 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=143616 expected_recv=256940 expected_overflow=0 exact_recv_per_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=143616 expected_recv=256940 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=143616 expected_recv=262163 expected_overflow=0 exact_recv_per_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=259328 slack_slots=27904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=143616 expected_recv=263492 expected_overflow=0 exact_recv_per_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=143616 expected_recv=261787 expected_overflow=0 exact_recv_per_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=143616 expected_recv=262163 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=143616 expected_recv=261937 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=143616 expected_recv=264452 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=143616 expected_recv=258474 expected_overflow=0 exact_recv_per_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=143616 expected_recv=263492 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=143616 expected_recv=261787 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=143616 expected_recv=261937 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=143616 expected_recv=264452 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=143616 expected_recv=258474 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=265472 slack_slots=21760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=143616 expected_recv=263865 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=143616 expected_recv=263865 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=265856 slack_slots=21376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=143616 expected_recv=260762 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=143616 expected_recv=260762 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=143616 expected_recv=247834 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=262912 slack_slots=24320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=143616 expected_recv=247834 actual_valid_recv=250112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=250112 slack_slots=37120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=143616 expected_recv=259273 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=143616 expected_recv=259273 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=143616 expected_recv=262335 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=143616 expected_recv=258803 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=143616 expected_recv=261679 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=143616 expected_recv=262335 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=143616 expected_recv=258803 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=143616 expected_recv=260263 expected_overflow=0 exact_recv_per_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=143616 expected_recv=261679 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=143616 expected_recv=260263 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=143616 expected_recv=255892 expected_overflow=0 exact_recv_per_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=143616 expected_recv=259042 expected_overflow=0 exact_recv_per_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=143616 expected_recv=255892 actual_valid_recv=258176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=143616 expected_recv=259042 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=143616 expected_recv=262684 expected_overflow=0 exact_recv_per_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=143616 expected_recv=262232 expected_overflow=0 exact_recv_per_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=258176 slack_slots=29056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=143616 expected_recv=261785 expected_overflow=0 exact_recv_per_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=143616 expected_recv=262684 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=143616 expected_recv=262232 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=264960 slack_slots=22272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=143616 expected_recv=261785 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=143616 expected_recv=262714 expected_overflow=0 exact_recv_per_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=143616 expected_recv=262714 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=143616 expected_recv=261782 expected_overflow=0 exact_recv_per_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=143616 expected_recv=261782 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=143616 expected_recv=257453 expected_overflow=0 exact_recv_per_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=143616 expected_recv=258879 expected_overflow=0 exact_recv_per_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=143616 expected_recv=257453 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=259456 slack_slots=27776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=143616 expected_recv=258879 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=143616 expected_recv=259715 expected_overflow=0 exact_recv_per_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=143616 expected_recv=259715 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=143616 expected_recv=261557 expected_overflow=0 exact_recv_per_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[146776, 155205, 118650, 125810, 141957, 94174, 114682, 151322] avg=131072.0000 max=155205 max_rank=1 min=94174 min_rank=5 imbalance_ratio=1.184120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=143616 expected_recv=261557 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=143616 expected_recv=255034 expected_overflow=0 exact_recv_per_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=143616 expected_recv=255034 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=257408 slack_slots=29824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[146776, 155205, 118650, 125810, 141957, 94174, 114682, 151322] avg=131072.0000 max=155205 max_rank=1 min=94174 min_rank=5 imbalance_ratio=1.184120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=125810 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=22822 max_load_reduction_pct=0.147044 moved_to_replicas=127812 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[146776, 155205, 118650, 125810, 141957, 94174, 114682, 151322] avg=131072.0000 max=155205 max_rank=1 min=94174 min_rank=5 imbalance_ratio=1.184120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=125810 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=22822 max_load_reduction_pct=0.147044 moved_to_replicas=127812 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[146776, 155205, 118650, 125810, 141957, 94174, 114682, 151322] avg=131072.0000 max=155205 max_rank=1 min=94174 min_rank=5 imbalance_ratio=1.184120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[146776, 155205, 118650, 125810, 141957, 94174, 114682, 151322] avg=131072.0000 max=155205 max_rank=1 min=94174 min_rank=5 imbalance_ratio=1.184120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[98576, 245878, 131016, 115088, 57718, 134492, 151481, 114327] avg=131072.0000 max=245878 max_rank=1 min=57718 min_rank=4 imbalance_ratio=1.875900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=125810 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=22822 max_load_reduction_pct=0.147044 moved_to_replicas=127812 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[146776, 155205, 118650, 125810, 141957, 94174, 114682, 151322] avg=131072.0000 max=155205 max_rank=1 min=94174 min_rank=5 imbalance_ratio=1.184120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[146776, 155205, 118650, 125810, 141957, 94174, 114682, 151322] avg=131072.0000 max=155205 max_rank=1 min=94174 min_rank=5 imbalance_ratio=1.184120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=125810 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=22822 max_load_reduction_pct=0.147044 moved_to_replicas=127812 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[146776, 155205, 118650, 125810, 141957, 94174, 114682, 151322] avg=131072.0000 max=155205 max_rank=1 min=94174 min_rank=5 imbalance_ratio=1.184120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[98576, 245878, 131016, 115088, 57718, 134492, 151481, 114327] avg=131072.0000 max=245878 max_rank=1 min=57718 min_rank=4 imbalance_ratio=1.875900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=125810 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=22822 max_load_reduction_pct=0.147044 moved_to_replicas=127812 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] avg=131072.0000 max=132384 max_rank=1 min=127113 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=113494 max_load_reduction_pct=0.461587 moved_to_replicas=269465 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=125810 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=22822 max_load_reduction_pct=0.147044 moved_to_replicas=127812 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=125810 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=22822 max_load_reduction_pct=0.147044 moved_to_replicas=127812 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[98576, 245878, 131016, 115088, 57718, 134492, 151481, 114327] avg=131072.0000 max=245878 max_rank=1 min=57718 min_rank=4 imbalance_ratio=1.875900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] avg=131072.0000 max=132384 max_rank=1 min=127113 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=113494 max_load_reduction_pct=0.461587 moved_to_replicas=269465 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[132383, 132383, 128468, 125810, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=125810 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=22822 max_load_reduction_pct=0.147044 moved_to_replicas=127812 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[98576, 245878, 131016, 115088, 57718, 134492, 151481, 114327] avg=131072.0000 max=245878 max_rank=1 min=57718 min_rank=4 imbalance_ratio=1.875900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[98576, 245878, 131016, 115088, 57718, 134492, 151481, 114327] avg=131072.0000 max=245878 max_rank=1 min=57718 min_rank=4 imbalance_ratio=1.875900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[140218, 247962, 195332, 83977, 98788, 131685, 88715, 61899] avg=131072.0000 max=247962 max_rank=1 min=61899 min_rank=7 imbalance_ratio=1.891800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] avg=131072.0000 max=132384 max_rank=1 min=127113 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=113494 max_load_reduction_pct=0.461587 moved_to_replicas=269465 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[98576, 245878, 131016, 115088, 57718, 134492, 151481, 114327] avg=131072.0000 max=245878 max_rank=1 min=57718 min_rank=4 imbalance_ratio=1.875900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] avg=131072.0000 max=132384 max_rank=1 min=127113 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=113494 max_load_reduction_pct=0.461587 moved_to_replicas=269465 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[98576, 245878, 131016, 115088, 57718, 134492, 151481, 114327] avg=131072.0000 max=245878 max_rank=1 min=57718 min_rank=4 imbalance_ratio=1.875900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[140218, 247962, 195332, 83977, 98788, 131685, 88715, 61899] avg=131072.0000 max=247962 max_rank=1 min=61899 min_rank=7 imbalance_ratio=1.891800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[98576, 245878, 131016, 115088, 57718, 134492, 151481, 114327] avg=131072.0000 max=245878 max_rank=1 min=57718 min_rank=4 imbalance_ratio=1.875900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] avg=131072.0000 max=132384 max_rank=1 min=126518 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=115578 max_load_reduction_pct=0.466112 moved_to_replicas=381430 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] avg=131072.0000 max=132384 max_rank=1 min=127113 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=113494 max_load_reduction_pct=0.461587 moved_to_replicas=269465 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] avg=131072.0000 max=132384 max_rank=1 min=127113 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=113494 max_load_reduction_pct=0.461587 moved_to_replicas=269465 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] avg=131072.0000 max=132384 max_rank=1 min=127113 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=113494 max_load_reduction_pct=0.461587 moved_to_replicas=269465 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] avg=131072.0000 max=132384 max_rank=1 min=126518 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=115578 max_load_reduction_pct=0.466112 moved_to_replicas=381430 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[140218, 247962, 195332, 83977, 98788, 131685, 88715, 61899] avg=131072.0000 max=247962 max_rank=1 min=61899 min_rank=7 imbalance_ratio=1.891800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[131312, 132384, 131016, 132383, 127113, 132383, 132383, 129602] avg=131072.0000 max=132384 max_rank=1 min=127113 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=113494 max_load_reduction_pct=0.461587 moved_to_replicas=269465 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[140218, 247962, 195332, 83977, 98788, 131685, 88715, 61899] avg=131072.0000 max=247962 max_rank=1 min=61899 min_rank=7 imbalance_ratio=1.891800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[121218, 78553, 221060, 169319, 99482, 195810, 64861, 98273] avg=131072.0000 max=221060 max_rank=2 min=64861 min_rank=6 imbalance_ratio=1.686554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[140218, 247962, 195332, 83977, 98788, 131685, 88715, 61899] avg=131072.0000 max=247962 max_rank=1 min=61899 min_rank=7 imbalance_ratio=1.891800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[140218, 247962, 195332, 83977, 98788, 131685, 88715, 61899] avg=131072.0000 max=247962 max_rank=1 min=61899 min_rank=7 imbalance_ratio=1.891800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] avg=131072.0000 max=132384 max_rank=1 min=126518 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=115578 max_load_reduction_pct=0.466112 moved_to_replicas=381430 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[140218, 247962, 195332, 83977, 98788, 131685, 88715, 61899] avg=131072.0000 max=247962 max_rank=1 min=61899 min_rank=7 imbalance_ratio=1.891800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] avg=131072.0000 max=132384 max_rank=1 min=126518 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=115578 max_load_reduction_pct=0.466112 moved_to_replicas=381430 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[121218, 78553, 221060, 169319, 99482, 195810, 64861, 98273] avg=131072.0000 max=221060 max_rank=2 min=64861 min_rank=6 imbalance_ratio=1.686554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] avg=131072.0000 max=132384 max_rank=6 min=125480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=88676 max_load_reduction_pct=0.401140 moved_to_replicas=381041 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] avg=131072.0000 max=132384 max_rank=1 min=126518 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=115578 max_load_reduction_pct=0.466112 moved_to_replicas=381430 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] avg=131072.0000 max=132384 max_rank=1 min=126518 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=115578 max_load_reduction_pct=0.466112 moved_to_replicas=381430 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[140218, 247962, 195332, 83977, 98788, 131685, 88715, 61899] avg=131072.0000 max=247962 max_rank=1 min=61899 min_rank=7 imbalance_ratio=1.891800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] avg=131072.0000 max=132384 max_rank=1 min=126518 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=115578 max_load_reduction_pct=0.466112 moved_to_replicas=381430 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] avg=131072.0000 max=132384 max_rank=6 min=125480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=88676 max_load_reduction_pct=0.401140 moved_to_replicas=381041 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[121218, 78553, 221060, 169319, 99482, 195810, 64861, 98273] avg=131072.0000 max=221060 max_rank=2 min=64861 min_rank=6 imbalance_ratio=1.686554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[121218, 78553, 221060, 169319, 99482, 195810, 64861, 98273] avg=131072.0000 max=221060 max_rank=2 min=64861 min_rank=6 imbalance_ratio=1.686554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[132382, 132384, 132383, 126518, 131544, 131685, 129298, 132382] avg=131072.0000 max=132384 max_rank=1 min=126518 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=115578 max_load_reduction_pct=0.466112 moved_to_replicas=381430 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32543, 98288, 108689, 196601, 219077, 98447, 130685, 164246] avg=131072.0000 max=219077 max_rank=4 min=32543 min_rank=0 imbalance_ratio=1.671425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[121218, 78553, 221060, 169319, 99482, 195810, 64861, 98273] avg=131072.0000 max=221060 max_rank=2 min=64861 min_rank=6 imbalance_ratio=1.686554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[121218, 78553, 221060, 169319, 99482, 195810, 64861, 98273] avg=131072.0000 max=221060 max_rank=2 min=64861 min_rank=6 imbalance_ratio=1.686554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] avg=131072.0000 max=132384 max_rank=6 min=125480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=88676 max_load_reduction_pct=0.401140 moved_to_replicas=381041 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[121218, 78553, 221060, 169319, 99482, 195810, 64861, 98273] avg=131072.0000 max=221060 max_rank=2 min=64861 min_rank=6 imbalance_ratio=1.686554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] avg=131072.0000 max=132384 max_rank=6 min=125480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=88676 max_load_reduction_pct=0.401140 moved_to_replicas=381041 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32543, 98288, 108689, 196601, 219077, 98447, 130685, 164246] avg=131072.0000 max=219077 max_rank=4 min=32543 min_rank=0 imbalance_ratio=1.671425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] avg=131072.0000 max=132383 max_rank=3 min=127407 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=86694 max_load_reduction_pct=0.395724 moved_to_replicas=370796 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] avg=131072.0000 max=132384 max_rank=6 min=125480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=88676 max_load_reduction_pct=0.401140 moved_to_replicas=381041 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] avg=131072.0000 max=132384 max_rank=6 min=125480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=88676 max_load_reduction_pct=0.401140 moved_to_replicas=381041 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[121218, 78553, 221060, 169319, 99482, 195810, 64861, 98273] avg=131072.0000 max=221060 max_rank=2 min=64861 min_rank=6 imbalance_ratio=1.686554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] avg=131072.0000 max=132384 max_rank=6 min=125480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=88676 max_load_reduction_pct=0.401140 moved_to_replicas=381041 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] avg=131072.0000 max=132383 max_rank=3 min=127407 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=86694 max_load_reduction_pct=0.395724 moved_to_replicas=370796 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32543, 98288, 108689, 196601, 219077, 98447, 130685, 164246] avg=131072.0000 max=219077 max_rank=4 min=32543 min_rank=0 imbalance_ratio=1.671425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32543, 98288, 108689, 196601, 219077, 98447, 130685, 164246] avg=131072.0000 max=219077 max_rank=4 min=32543 min_rank=0 imbalance_ratio=1.671425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[76457, 131056, 231488, 149916, 66631, 163828, 98258, 130942] avg=131072.0000 max=231488 max_rank=2 min=66631 min_rank=4 imbalance_ratio=1.766113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[125480, 132383, 132382, 132382, 130153, 132383, 132384, 131029] avg=131072.0000 max=132384 max_rank=6 min=125480 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=88676 max_load_reduction_pct=0.401140 moved_to_replicas=381041 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32543, 98288, 108689, 196601, 219077, 98447, 130685, 164246] avg=131072.0000 max=219077 max_rank=4 min=32543 min_rank=0 imbalance_ratio=1.671425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] avg=131072.0000 max=132383 max_rank=3 min=127407 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=86694 max_load_reduction_pct=0.395724 moved_to_replicas=370796 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32543, 98288, 108689, 196601, 219077, 98447, 130685, 164246] avg=131072.0000 max=219077 max_rank=4 min=32543 min_rank=0 imbalance_ratio=1.671425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32543, 98288, 108689, 196601, 219077, 98447, 130685, 164246] avg=131072.0000 max=219077 max_rank=4 min=32543 min_rank=0 imbalance_ratio=1.671425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] avg=131072.0000 max=132383 max_rank=3 min=127407 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=86694 max_load_reduction_pct=0.395724 moved_to_replicas=370796 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[76457, 131056, 231488, 149916, 66631, 163828, 98258, 130942] avg=131072.0000 max=231488 max_rank=2 min=66631 min_rank=4 imbalance_ratio=1.766113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] avg=131072.0000 max=132383 max_rank=2 min=126745 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=99105 max_load_reduction_pct=0.428122 moved_to_replicas=295867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] avg=131072.0000 max=132383 max_rank=3 min=127407 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=86694 max_load_reduction_pct=0.395724 moved_to_replicas=370796 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] avg=131072.0000 max=132383 max_rank=3 min=127407 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=86694 max_load_reduction_pct=0.395724 moved_to_replicas=370796 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] avg=131072.0000 max=132383 max_rank=3 min=127407 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=86694 max_load_reduction_pct=0.395724 moved_to_replicas=370796 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32543, 98288, 108689, 196601, 219077, 98447, 130685, 164246] avg=131072.0000 max=219077 max_rank=4 min=32543 min_rank=0 imbalance_ratio=1.671425 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] avg=131072.0000 max=132383 max_rank=2 min=126745 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=99105 max_load_reduction_pct=0.428122 moved_to_replicas=295867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[76457, 131056, 231488, 149916, 66631, 163828, 98258, 130942] avg=131072.0000 max=231488 max_rank=2 min=66631 min_rank=4 imbalance_ratio=1.766113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[76457, 131056, 231488, 149916, 66631, 163828, 98258, 130942] avg=131072.0000 max=231488 max_rank=2 min=66631 min_rank=4 imbalance_ratio=1.766113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65769, 111489, 129994, 201508, 147286, 229197, 97741, 65592] avg=131072.0000 max=229197 max_rank=5 min=65592 min_rank=7 imbalance_ratio=1.748634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[127407, 131044, 132382, 132383, 132383, 129909, 130685, 132383] avg=131072.0000 max=132383 max_rank=3 min=127407 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=86694 max_load_reduction_pct=0.395724 moved_to_replicas=370796 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[76457, 131056, 231488, 149916, 66631, 163828, 98258, 130942] avg=131072.0000 max=231488 max_rank=2 min=66631 min_rank=4 imbalance_ratio=1.766113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] avg=131072.0000 max=132383 max_rank=2 min=126745 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=99105 max_load_reduction_pct=0.428122 moved_to_replicas=295867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[76457, 131056, 231488, 149916, 66631, 163828, 98258, 130942] avg=131072.0000 max=231488 max_rank=2 min=66631 min_rank=4 imbalance_ratio=1.766113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[76457, 131056, 231488, 149916, 66631, 163828, 98258, 130942] avg=131072.0000 max=231488 max_rank=2 min=66631 min_rank=4 imbalance_ratio=1.766113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] avg=131072.0000 max=132383 max_rank=2 min=126745 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=99105 max_load_reduction_pct=0.428122 moved_to_replicas=295867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65769, 111489, 129994, 201508, 147286, 229197, 97741, 65592] avg=131072.0000 max=229197 max_rank=5 min=65592 min_rank=7 imbalance_ratio=1.748634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] avg=131072.0000 max=132383 max_rank=3 min=129049 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96814 max_load_reduction_pct=0.422405 moved_to_replicas=352604 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] avg=131072.0000 max=132383 max_rank=2 min=126745 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=99105 max_load_reduction_pct=0.428122 moved_to_replicas=295867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] avg=131072.0000 max=132383 max_rank=2 min=126745 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=99105 max_load_reduction_pct=0.428122 moved_to_replicas=295867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] avg=131072.0000 max=132383 max_rank=2 min=126745 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=99105 max_load_reduction_pct=0.428122 moved_to_replicas=295867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[76457, 131056, 231488, 149916, 66631, 163828, 98258, 130942] avg=131072.0000 max=231488 max_rank=2 min=66631 min_rank=4 imbalance_ratio=1.766113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] avg=131072.0000 max=132383 max_rank=3 min=129049 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96814 max_load_reduction_pct=0.422405 moved_to_replicas=352604 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65769, 111489, 129994, 201508, 147286, 229197, 97741, 65592] avg=131072.0000 max=229197 max_rank=5 min=65592 min_rank=7 imbalance_ratio=1.748634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65769, 111489, 129994, 201508, 147286, 229197, 97741, 65592] avg=131072.0000 max=229197 max_rank=5 min=65592 min_rank=7 imbalance_ratio=1.748634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[97326, 65661, 162940, 131077, 161996, 229103, 65578, 134895] avg=131072.0000 max=229103 max_rank=5 min=65578 min_rank=6 imbalance_ratio=1.747917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[126745, 131056, 132383, 132382, 131674, 132383, 131011, 130942] avg=131072.0000 max=132383 max_rank=2 min=126745 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=99105 max_load_reduction_pct=0.428122 moved_to_replicas=295867 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] avg=131072.0000 max=132383 max_rank=3 min=129049 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96814 max_load_reduction_pct=0.422405 moved_to_replicas=352604 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65769, 111489, 129994, 201508, 147286, 229197, 97741, 65592] avg=131072.0000 max=229197 max_rank=5 min=65592 min_rank=7 imbalance_ratio=1.748634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65769, 111489, 129994, 201508, 147286, 229197, 97741, 65592] avg=131072.0000 max=229197 max_rank=5 min=65592 min_rank=7 imbalance_ratio=1.748634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] avg=131072.0000 max=132383 max_rank=3 min=129049 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96814 max_load_reduction_pct=0.422405 moved_to_replicas=352604 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65769, 111489, 129994, 201508, 147286, 229197, 97741, 65592] avg=131072.0000 max=229197 max_rank=5 min=65592 min_rank=7 imbalance_ratio=1.748634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[97326, 65661, 162940, 131077, 161996, 229103, 65578, 134895] avg=131072.0000 max=229103 max_rank=5 min=65578 min_rank=6 imbalance_ratio=1.747917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] avg=131072.0000 max=132383 max_rank=2 min=128538 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96720 max_load_reduction_pct=0.422168 moved_to_replicas=318865 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] avg=131072.0000 max=132383 max_rank=3 min=129049 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96814 max_load_reduction_pct=0.422405 moved_to_replicas=352604 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] avg=131072.0000 max=132383 max_rank=3 min=129049 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96814 max_load_reduction_pct=0.422405 moved_to_replicas=352604 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65769, 111489, 129994, 201508, 147286, 229197, 97741, 65592] avg=131072.0000 max=229197 max_rank=5 min=65592 min_rank=7 imbalance_ratio=1.748634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] avg=131072.0000 max=132383 max_rank=3 min=129049 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96814 max_load_reduction_pct=0.422405 moved_to_replicas=352604 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] avg=131072.0000 max=132383 max_rank=2 min=128538 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96720 max_load_reduction_pct=0.422168 moved_to_replicas=318865 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[97326, 65661, 162940, 131077, 161996, 229103, 65578, 134895] avg=131072.0000 max=229103 max_rank=5 min=65578 min_rank=6 imbalance_ratio=1.747917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[97326, 65661, 162940, 131077, 161996, 229103, 65578, 134895] avg=131072.0000 max=229103 max_rank=5 min=65578 min_rank=6 imbalance_ratio=1.747917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[51101, 146278, 98322, 130877, 131073, 32868, 228740, 229317] avg=131072.0000 max=229317 max_rank=7 min=32868 min_rank=5 imbalance_ratio=1.749550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[131274, 130009, 129994, 132383, 132383, 132383, 129049, 131101] avg=131072.0000 max=132383 max_rank=3 min=129049 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96814 max_load_reduction_pct=0.422405 moved_to_replicas=352604 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[97326, 65661, 162940, 131077, 161996, 229103, 65578, 134895] avg=131072.0000 max=229103 max_rank=5 min=65578 min_rank=6 imbalance_ratio=1.747917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] avg=131072.0000 max=132383 max_rank=2 min=128538 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96720 max_load_reduction_pct=0.422168 moved_to_replicas=318865 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[97326, 65661, 162940, 131077, 161996, 229103, 65578, 134895] avg=131072.0000 max=229103 max_rank=5 min=65578 min_rank=6 imbalance_ratio=1.747917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] avg=131072.0000 max=132383 max_rank=2 min=128538 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96720 max_load_reduction_pct=0.422168 moved_to_replicas=318865 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[97326, 65661, 162940, 131077, 161996, 229103, 65578, 134895] avg=131072.0000 max=229103 max_rank=5 min=65578 min_rank=6 imbalance_ratio=1.747917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[51101, 146278, 98322, 130877, 131073, 32868, 228740, 229317] avg=131072.0000 max=229317 max_rank=7 min=32868 min_rank=5 imbalance_ratio=1.749550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] avg=131072.0000 max=132383 max_rank=6 min=129175 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96934 max_load_reduction_pct=0.422707 moved_to_replicas=413532 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] avg=131072.0000 max=132383 max_rank=2 min=128538 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96720 max_load_reduction_pct=0.422168 moved_to_replicas=318865 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] avg=131072.0000 max=132383 max_rank=2 min=128538 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96720 max_load_reduction_pct=0.422168 moved_to_replicas=318865 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[97326, 65661, 162940, 131077, 161996, 229103, 65578, 134895] avg=131072.0000 max=229103 max_rank=5 min=65578 min_rank=6 imbalance_ratio=1.747917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] avg=131072.0000 max=132383 max_rank=2 min=128538 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96720 max_load_reduction_pct=0.422168 moved_to_replicas=318865 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] avg=131072.0000 max=132383 max_rank=6 min=129175 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96934 max_load_reduction_pct=0.422707 moved_to_replicas=413532 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[51101, 146278, 98322, 130877, 131073, 32868, 228740, 229317] avg=131072.0000 max=229317 max_rank=7 min=32868 min_rank=5 imbalance_ratio=1.749550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[51101, 146278, 98322, 130877, 131073, 32868, 228740, 229317] avg=131072.0000 max=229317 max_rank=7 min=32868 min_rank=5 imbalance_ratio=1.749550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[139300, 92382, 98386, 131050, 98157, 161469, 131233, 196599] avg=131072.0000 max=196599 max_rank=7 min=92382 min_rank=1 imbalance_ratio=1.499931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[128538, 130540, 132383, 131077, 132383, 132383, 128889, 132383] avg=131072.0000 max=132383 max_rank=2 min=128538 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96720 max_load_reduction_pct=0.422168 moved_to_replicas=318865 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[51101, 146278, 98322, 130877, 131073, 32868, 228740, 229317] avg=131072.0000 max=229317 max_rank=7 min=32868 min_rank=5 imbalance_ratio=1.749550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[51101, 146278, 98322, 130877, 131073, 32868, 228740, 229317] avg=131072.0000 max=229317 max_rank=7 min=32868 min_rank=5 imbalance_ratio=1.749550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] avg=131072.0000 max=132383 max_rank=6 min=129175 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96934 max_load_reduction_pct=0.422707 moved_to_replicas=413532 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] avg=131072.0000 max=132383 max_rank=6 min=129175 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96934 max_load_reduction_pct=0.422707 moved_to_replicas=413532 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] avg=131072.0000 max=132385 max_rank=0 min=127472 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=64214 max_load_reduction_pct=0.326624 moved_to_replicas=200676 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[139300, 92382, 98386, 131050, 98157, 161469, 131233, 196599] avg=131072.0000 max=196599 max_rank=7 min=92382 min_rank=1 imbalance_ratio=1.499931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[51101, 146278, 98322, 130877, 131073, 32868, 228740, 229317] avg=131072.0000 max=229317 max_rank=7 min=32868 min_rank=5 imbalance_ratio=1.749550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] avg=131072.0000 max=132383 max_rank=6 min=129175 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96934 max_load_reduction_pct=0.422707 moved_to_replicas=413532 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] avg=131072.0000 max=132383 max_rank=6 min=129175 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96934 max_load_reduction_pct=0.422707 moved_to_replicas=413532 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[51101, 146278, 98322, 130877, 131073, 32868, 228740, 229317] avg=131072.0000 max=229317 max_rank=7 min=32868 min_rank=5 imbalance_ratio=1.749550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] avg=131072.0000 max=132385 max_rank=0 min=127472 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=64214 max_load_reduction_pct=0.326624 moved_to_replicas=200676 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[139300, 92382, 98386, 131050, 98157, 161469, 131233, 196599] avg=131072.0000 max=196599 max_rank=7 min=92382 min_rank=1 imbalance_ratio=1.499931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[139300, 92382, 98386, 131050, 98157, 161469, 131233, 196599] avg=131072.0000 max=196599 max_rank=7 min=92382 min_rank=1 imbalance_ratio=1.499931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] avg=131072.0000 max=132383 max_rank=6 min=129175 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96934 max_load_reduction_pct=0.422707 moved_to_replicas=413532 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[98570, 215022, 61060, 164409, 98360, 214598, 65586, 130971] avg=131072.0000 max=215022 max_rank=1 min=61060 min_rank=2 imbalance_ratio=1.640488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[130506, 132382, 129175, 130877, 131073, 129797, 132383, 132383] avg=131072.0000 max=132383 max_rank=6 min=129175 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96934 max_load_reduction_pct=0.422707 moved_to_replicas=413532 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[139300, 92382, 98386, 131050, 98157, 161469, 131233, 196599] avg=131072.0000 max=196599 max_rank=7 min=92382 min_rank=1 imbalance_ratio=1.499931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] avg=131072.0000 max=132385 max_rank=0 min=127472 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=64214 max_load_reduction_pct=0.326624 moved_to_replicas=200676 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[139300, 92382, 98386, 131050, 98157, 161469, 131233, 196599] avg=131072.0000 max=196599 max_rank=7 min=92382 min_rank=1 imbalance_ratio=1.499931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] avg=131072.0000 max=132385 max_rank=0 min=127472 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=64214 max_load_reduction_pct=0.326624 moved_to_replicas=200676 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] avg=131072.0000 max=132384 max_rank=2 min=125871 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=82638 max_load_reduction_pct=0.384323 moved_to_replicas=392510 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[98570, 215022, 61060, 164409, 98360, 214598, 65586, 130971] avg=131072.0000 max=215022 max_rank=1 min=61060 min_rank=2 imbalance_ratio=1.640488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[139300, 92382, 98386, 131050, 98157, 161469, 131233, 196599] avg=131072.0000 max=196599 max_rank=7 min=92382 min_rank=1 imbalance_ratio=1.499931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] avg=131072.0000 max=132385 max_rank=0 min=127472 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=64214 max_load_reduction_pct=0.326624 moved_to_replicas=200676 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] avg=131072.0000 max=132385 max_rank=0 min=127472 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=64214 max_load_reduction_pct=0.326624 moved_to_replicas=200676 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] avg=131072.0000 max=132384 max_rank=2 min=125871 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=82638 max_load_reduction_pct=0.384323 moved_to_replicas=392510 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[139300, 92382, 98386, 131050, 98157, 161469, 131233, 196599] avg=131072.0000 max=196599 max_rank=7 min=92382 min_rank=1 imbalance_ratio=1.499931 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[98570, 215022, 61060, 164409, 98360, 214598, 65586, 130971] avg=131072.0000 max=215022 max_rank=1 min=61060 min_rank=2 imbalance_ratio=1.640488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] avg=131072.0000 max=132385 max_rank=0 min=127472 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=64214 max_load_reduction_pct=0.326624 moved_to_replicas=200676 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[100365, 133511, 56412, 130883, 153761, 240698, 167335, 65611] avg=131072.0000 max=240698 max_rank=5 min=56412 min_rank=2 imbalance_ratio=1.836380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[98570, 215022, 61060, 164409, 98360, 214598, 65586, 130971] avg=131072.0000 max=215022 max_rank=1 min=61060 min_rank=2 imbalance_ratio=1.640488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[132385, 132052, 127472, 131050, 129618, 132383, 131233, 132383] avg=131072.0000 max=132385 max_rank=0 min=127472 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=64214 max_load_reduction_pct=0.326624 moved_to_replicas=200676 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[98570, 215022, 61060, 164409, 98360, 214598, 65586, 130971] avg=131072.0000 max=215022 max_rank=1 min=61060 min_rank=2 imbalance_ratio=1.640488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] avg=131072.0000 max=132384 max_rank=2 min=125871 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=82638 max_load_reduction_pct=0.384323 moved_to_replicas=392510 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[98570, 215022, 61060, 164409, 98360, 214598, 65586, 130971] avg=131072.0000 max=215022 max_rank=1 min=61060 min_rank=2 imbalance_ratio=1.640488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[100365, 133511, 56412, 130883, 153761, 240698, 167335, 65611] avg=131072.0000 max=240698 max_rank=5 min=56412 min_rank=2 imbalance_ratio=1.836380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=123395 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108313 max_load_reduction_pct=0.449995 moved_to_replicas=334754 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] avg=131072.0000 max=132384 max_rank=2 min=125871 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=82638 max_load_reduction_pct=0.384323 moved_to_replicas=392510 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[98570, 215022, 61060, 164409, 98360, 214598, 65586, 130971] avg=131072.0000 max=215022 max_rank=1 min=61060 min_rank=2 imbalance_ratio=1.640488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] avg=131072.0000 max=132384 max_rank=2 min=125871 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=82638 max_load_reduction_pct=0.384323 moved_to_replicas=392510 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] avg=131072.0000 max=132384 max_rank=2 min=125871 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=82638 max_load_reduction_pct=0.384323 moved_to_replicas=392510 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=123395 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108313 max_load_reduction_pct=0.449995 moved_to_replicas=334754 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[100365, 133511, 56412, 130883, 153761, 240698, 167335, 65611] avg=131072.0000 max=240698 max_rank=5 min=56412 min_rank=2 imbalance_ratio=1.836380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[98570, 215022, 61060, 164409, 98360, 214598, 65586, 130971] avg=131072.0000 max=215022 max_rank=1 min=61060 min_rank=2 imbalance_ratio=1.640488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] avg=131072.0000 max=132384 max_rank=2 min=125871 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=82638 max_load_reduction_pct=0.384323 moved_to_replicas=392510 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[225880, 165203, 66029, 98430, 133322, 98700, 129778, 131234] avg=131072.0000 max=225880 max_rank=0 min=66029 min_rank=2 imbalance_ratio=1.723328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[100365, 133511, 56412, 130883, 153761, 240698, 167335, 65611] avg=131072.0000 max=240698 max_rank=5 min=56412 min_rank=2 imbalance_ratio=1.836380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[100365, 133511, 56412, 130883, 153761, 240698, 167335, 65611] avg=131072.0000 max=240698 max_rank=5 min=56412 min_rank=2 imbalance_ratio=1.836380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[100365, 133511, 56412, 130883, 153761, 240698, 167335, 65611] avg=131072.0000 max=240698 max_rank=5 min=56412 min_rank=2 imbalance_ratio=1.836380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[225880, 165203, 66029, 98430, 133322, 98700, 129778, 131234] avg=131072.0000 max=225880 max_rank=0 min=66029 min_rank=2 imbalance_ratio=1.723328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=123395 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108313 max_load_reduction_pct=0.449995 moved_to_replicas=334754 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[125871, 132382, 132384, 132383, 131109, 132383, 131093, 130971] avg=131072.0000 max=132384 max_rank=2 min=125871 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=82638 max_load_reduction_pct=0.384323 moved_to_replicas=392510 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] avg=131072.0000 max=132385 max_rank=0 min=127701 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=93495 max_load_reduction_pct=0.413914 moved_to_replicas=255493 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=123395 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108313 max_load_reduction_pct=0.449995 moved_to_replicas=334754 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=123395 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108313 max_load_reduction_pct=0.449995 moved_to_replicas=334754 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[100365, 133511, 56412, 130883, 153761, 240698, 167335, 65611] avg=131072.0000 max=240698 max_rank=5 min=56412 min_rank=2 imbalance_ratio=1.836380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=123395 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108313 max_load_reduction_pct=0.449995 moved_to_replicas=334754 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] avg=131072.0000 max=132385 max_rank=0 min=127701 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=93495 max_load_reduction_pct=0.413914 moved_to_replicas=255493 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[225880, 165203, 66029, 98430, 133322, 98700, 129778, 131234] avg=131072.0000 max=225880 max_rank=0 min=66029 min_rank=2 imbalance_ratio=1.723328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[66284, 110660, 130741, 163815, 123915, 229271, 125493, 98397] avg=131072.0000 max=229271 max_rank=5 min=66284 min_rank=0 imbalance_ratio=1.749199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[100365, 133511, 56412, 130883, 153761, 240698, 167335, 65611] avg=131072.0000 max=240698 max_rank=5 min=56412 min_rank=2 imbalance_ratio=1.836380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[225880, 165203, 66029, 98430, 133322, 98700, 129778, 131234] avg=131072.0000 max=225880 max_rank=0 min=66029 min_rank=2 imbalance_ratio=1.723328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=123395 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108313 max_load_reduction_pct=0.449995 moved_to_replicas=334754 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[225880, 165203, 66029, 98430, 133322, 98700, 129778, 131234] avg=131072.0000 max=225880 max_rank=0 min=66029 min_rank=2 imbalance_ratio=1.723328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[225880, 165203, 66029, 98430, 133322, 98700, 129778, 131234] avg=131072.0000 max=225880 max_rank=0 min=66029 min_rank=2 imbalance_ratio=1.723328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[66284, 110660, 130741, 163815, 123915, 229271, 125493, 98397] avg=131072.0000 max=229271 max_rank=5 min=66284 min_rank=0 imbalance_ratio=1.749199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] avg=131072.0000 max=132385 max_rank=0 min=127701 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=93495 max_load_reduction_pct=0.413914 moved_to_replicas=255493 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] avg=131072.0000 max=132383 max_rank=1 min=126734 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96888 max_load_reduction_pct=0.422592 moved_to_replicas=256748 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123395, 130883, 132383, 132381, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=123395 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108313 max_load_reduction_pct=0.449995 moved_to_replicas=334754 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] avg=131072.0000 max=132385 max_rank=0 min=127701 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=93495 max_load_reduction_pct=0.413914 moved_to_replicas=255493 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] avg=131072.0000 max=132385 max_rank=0 min=127701 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=93495 max_load_reduction_pct=0.413914 moved_to_replicas=255493 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] avg=131072.0000 max=132385 max_rank=0 min=127701 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=93495 max_load_reduction_pct=0.413914 moved_to_replicas=255493 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[225880, 165203, 66029, 98430, 133322, 98700, 129778, 131234] avg=131072.0000 max=225880 max_rank=0 min=66029 min_rank=2 imbalance_ratio=1.723328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] avg=131072.0000 max=132383 max_rank=1 min=126734 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96888 max_load_reduction_pct=0.422592 moved_to_replicas=256748 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[66284, 110660, 130741, 163815, 123915, 229271, 125493, 98397] avg=131072.0000 max=229271 max_rank=5 min=66284 min_rank=0 imbalance_ratio=1.749199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[131592, 197367, 33108, 65228, 163727, 163980, 196900, 96674] avg=131072.0000 max=197367 max_rank=1 min=33108 min_rank=2 imbalance_ratio=1.505791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[225880, 165203, 66029, 98430, 133322, 98700, 129778, 131234] avg=131072.0000 max=225880 max_rank=0 min=66029 min_rank=2 imbalance_ratio=1.723328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] avg=131072.0000 max=132385 max_rank=0 min=127701 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=93495 max_load_reduction_pct=0.413914 moved_to_replicas=255493 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[66284, 110660, 130741, 163815, 123915, 229271, 125493, 98397] avg=131072.0000 max=229271 max_rank=5 min=66284 min_rank=0 imbalance_ratio=1.749199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[66284, 110660, 130741, 163815, 123915, 229271, 125493, 98397] avg=131072.0000 max=229271 max_rank=5 min=66284 min_rank=0 imbalance_ratio=1.749199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[66284, 110660, 130741, 163815, 123915, 229271, 125493, 98397] avg=131072.0000 max=229271 max_rank=5 min=66284 min_rank=0 imbalance_ratio=1.749199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] avg=131072.0000 max=132383 max_rank=1 min=126734 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96888 max_load_reduction_pct=0.422592 moved_to_replicas=256748 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[131592, 197367, 33108, 65228, 163727, 163980, 196900, 96674] avg=131072.0000 max=197367 max_rank=1 min=33108 min_rank=2 imbalance_ratio=1.505791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] avg=131072.0000 max=132383 max_rank=1 min=128438 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64984 max_load_reduction_pct=0.329255 moved_to_replicas=384418 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[132385, 132383, 131531, 131181, 132383, 127701, 129778, 131234] avg=131072.0000 max=132385 max_rank=0 min=127701 min_rank=5 imbalance_ratio=1.010017 max_load_reduction=93495 max_load_reduction_pct=0.413914 moved_to_replicas=255493 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] avg=131072.0000 max=132383 max_rank=1 min=126734 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96888 max_load_reduction_pct=0.422592 moved_to_replicas=256748 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] avg=131072.0000 max=132383 max_rank=1 min=126734 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96888 max_load_reduction_pct=0.422592 moved_to_replicas=256748 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] avg=131072.0000 max=132383 max_rank=1 min=126734 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96888 max_load_reduction_pct=0.422592 moved_to_replicas=256748 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[66284, 110660, 130741, 163815, 123915, 229271, 125493, 98397] avg=131072.0000 max=229271 max_rank=5 min=66284 min_rank=0 imbalance_ratio=1.749199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] avg=131072.0000 max=132383 max_rank=1 min=128438 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64984 max_load_reduction_pct=0.329255 moved_to_replicas=384418 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[131592, 197367, 33108, 65228, 163727, 163980, 196900, 96674] avg=131072.0000 max=197367 max_rank=1 min=33108 min_rank=2 imbalance_ratio=1.505791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[98059, 164227, 69973, 130961, 126805, 163710, 130562, 164279] avg=131072.0000 max=164279 max_rank=7 min=69973 min_rank=2 imbalance_ratio=1.253349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[66284, 110660, 130741, 163815, 123915, 229271, 125493, 98397] avg=131072.0000 max=229271 max_rank=5 min=66284 min_rank=0 imbalance_ratio=1.749199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] avg=131072.0000 max=132383 max_rank=1 min=126734 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96888 max_load_reduction_pct=0.422592 moved_to_replicas=256748 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[131592, 197367, 33108, 65228, 163727, 163980, 196900, 96674] avg=131072.0000 max=197367 max_rank=1 min=33108 min_rank=2 imbalance_ratio=1.505791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[131592, 197367, 33108, 65228, 163727, 163980, 196900, 96674] avg=131072.0000 max=197367 max_rank=1 min=33108 min_rank=2 imbalance_ratio=1.505791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[131592, 197367, 33108, 65228, 163727, 163980, 196900, 96674] avg=131072.0000 max=197367 max_rank=1 min=33108 min_rank=2 imbalance_ratio=1.505791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[98059, 164227, 69973, 130961, 126805, 163710, 130562, 164279] avg=131072.0000 max=164279 max_rank=7 min=69973 min_rank=2 imbalance_ratio=1.253349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] avg=131072.0000 max=132383 max_rank=1 min=128438 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64984 max_load_reduction_pct=0.329255 moved_to_replicas=384418 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] avg=131072.0000 max=132383 max_rank=1 min=127618 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31896 max_load_reduction_pct=0.194158 moved_to_replicas=189557 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[130420, 132383, 130741, 132383, 132383, 132383, 126734, 131149] avg=131072.0000 max=132383 max_rank=1 min=126734 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96888 max_load_reduction_pct=0.422592 moved_to_replicas=256748 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] avg=131072.0000 max=132383 max_rank=1 min=128438 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64984 max_load_reduction_pct=0.329255 moved_to_replicas=384418 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] avg=131072.0000 max=132383 max_rank=1 min=128438 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64984 max_load_reduction_pct=0.329255 moved_to_replicas=384418 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] avg=131072.0000 max=132383 max_rank=1 min=128438 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64984 max_load_reduction_pct=0.329255 moved_to_replicas=384418 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] avg=131072.0000 max=132383 max_rank=1 min=127618 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31896 max_load_reduction_pct=0.194158 moved_to_replicas=189557 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[131592, 197367, 33108, 65228, 163727, 163980, 196900, 96674] avg=131072.0000 max=197367 max_rank=1 min=33108 min_rank=2 imbalance_ratio=1.505791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[98059, 164227, 69973, 130961, 126805, 163710, 130562, 164279] avg=131072.0000 max=164279 max_rank=7 min=69973 min_rank=2 imbalance_ratio=1.253349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[131592, 197367, 33108, 65228, 163727, 163980, 196900, 96674] avg=131072.0000 max=197367 max_rank=1 min=33108 min_rank=2 imbalance_ratio=1.505791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[98059, 164227, 69973, 130961, 126805, 163710, 130562, 164279] avg=131072.0000 max=164279 max_rank=7 min=69973 min_rank=2 imbalance_ratio=1.253349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] avg=131072.0000 max=132383 max_rank=1 min=128438 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64984 max_load_reduction_pct=0.329255 moved_to_replicas=384418 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[98059, 164227, 69973, 130961, 126805, 163710, 130562, 164279] avg=131072.0000 max=164279 max_rank=7 min=69973 min_rank=2 imbalance_ratio=1.253349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[98059, 164227, 69973, 130961, 126805, 163710, 130562, 164279] avg=131072.0000 max=164279 max_rank=7 min=69973 min_rank=2 imbalance_ratio=1.253349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] avg=131072.0000 max=132383 max_rank=1 min=127618 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31896 max_load_reduction_pct=0.194158 moved_to_replicas=189557 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[131592, 132383, 129960, 129054, 132383, 132383, 132383, 128438] avg=131072.0000 max=132383 max_rank=1 min=128438 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64984 max_load_reduction_pct=0.329255 moved_to_replicas=384418 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] avg=131072.0000 max=132383 max_rank=1 min=127618 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31896 max_load_reduction_pct=0.194158 moved_to_replicas=189557 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] avg=131072.0000 max=132383 max_rank=1 min=127618 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31896 max_load_reduction_pct=0.194158 moved_to_replicas=189557 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] avg=131072.0000 max=132383 max_rank=1 min=127618 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31896 max_load_reduction_pct=0.194158 moved_to_replicas=189557 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[98059, 164227, 69973, 130961, 126805, 163710, 130562, 164279] avg=131072.0000 max=164279 max_rank=7 min=69973 min_rank=2 imbalance_ratio=1.253349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[98059, 164227, 69973, 130961, 126805, 163710, 130562, 164279] avg=131072.0000 max=164279 max_rank=7 min=69973 min_rank=2 imbalance_ratio=1.253349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] avg=131072.0000 max=132383 max_rank=1 min=127618 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31896 max_load_reduction_pct=0.194158 moved_to_replicas=189557 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[129903, 132383, 132383, 130961, 127618, 132383, 130562, 132383] avg=131072.0000 max=132383 max_rank=1 min=127618 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31896 max_load_reduction_pct=0.194158 moved_to_replicas=189557 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=4191118 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=370176 peak_demand=132383 required_capacity=1.010002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=4218670 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=340864 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=4183762 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376320 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=4203882 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=358272 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=4216822 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=344704 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=4166612 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=394240 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=4204764 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=354304 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=4168802 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=388992 peak_demand=132384 required_capacity=1.010010 + [2026-08-30 05:19:11] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 571.6 | throughput per GPU (TFLOP/s/GPU): 583.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.323195E+00 | loss scale: 1.0 | grad norm: 5.519 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=143616 expected_recv=256641 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=143616 expected_recv=258291 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=143616 expected_recv=258392 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=143616 expected_recv=256641 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=143616 expected_recv=258291 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=259328 slack_slots=27904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=143616 expected_recv=258392 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=260608 slack_slots=26624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=143616 expected_recv=244951 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=143616 expected_recv=263607 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=143616 expected_recv=244951 actual_valid_recv=247040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=143616 expected_recv=263607 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=247040 slack_slots=40192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=143616 expected_recv=255825 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=143616 expected_recv=256859 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=143616 expected_recv=260633 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=143616 expected_recv=255825 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=143616 expected_recv=256859 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=143616 expected_recv=260633 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=258304 slack_slots=28928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267904 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=143616 expected_recv=264762 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=143616 expected_recv=264762 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=143616 expected_recv=257192 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=143616 expected_recv=260125 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=143616 expected_recv=257192 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=143616 expected_recv=260125 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=259968 slack_slots=27264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=143616 expected_recv=258283 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=143616 expected_recv=262492 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=143616 expected_recv=258283 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=143616 expected_recv=261916 expected_overflow=0 exact_recv_per_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=143616 expected_recv=262492 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=260864 slack_slots=26368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=143616 expected_recv=261916 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=143616 expected_recv=264316 expected_overflow=0 exact_recv_per_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=143616 expected_recv=264316 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=143616 expected_recv=262094 expected_overflow=0 exact_recv_per_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=143616 expected_recv=256914 expected_overflow=0 exact_recv_per_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=143616 expected_recv=259777 expected_overflow=0 exact_recv_per_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=143616 expected_recv=262606 expected_overflow=0 exact_recv_per_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=143616 expected_recv=262094 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=143616 expected_recv=262125 expected_overflow=0 exact_recv_per_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=143616 expected_recv=256914 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=143616 expected_recv=259777 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=143616 expected_recv=262606 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=259072 slack_slots=28160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=143616 expected_recv=262125 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=143616 expected_recv=261923 expected_overflow=0 exact_recv_per_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=143616 expected_recv=262989 expected_overflow=0 exact_recv_per_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=143616 expected_recv=261923 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=143616 expected_recv=253846 expected_overflow=0 exact_recv_per_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=143616 expected_recv=262989 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=143616 expected_recv=253846 actual_valid_recv=256000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=265600 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=143616 expected_recv=261973 expected_overflow=0 exact_recv_per_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=256000 slack_slots=31232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=143616 expected_recv=261973 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=143616 expected_recv=263083 expected_overflow=0 exact_recv_per_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=143616 expected_recv=263083 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=143616 expected_recv=264762 expected_overflow=0 exact_recv_per_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=143616 expected_recv=262014 expected_overflow=0 exact_recv_per_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=143616 expected_recv=264762 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=143616 expected_recv=262014 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=143616 expected_recv=251273 expected_overflow=0 exact_recv_per_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=143616 expected_recv=251273 actual_valid_recv=253440 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=143616 expected_recv=261719 expected_overflow=0 exact_recv_per_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=143616 expected_recv=256616 expected_overflow=0 exact_recv_per_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=253440 slack_slots=33792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=143616 expected_recv=260109 expected_overflow=0 exact_recv_per_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=143616 expected_recv=261719 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=143616 expected_recv=256616 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=143616 expected_recv=262220 expected_overflow=0 exact_recv_per_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=143616 expected_recv=260109 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=258304 slack_slots=28928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=143616 expected_recv=262220 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=143616 expected_recv=262281 expected_overflow=0 exact_recv_per_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=143616 expected_recv=262281 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=143616 expected_recv=261631 expected_overflow=0 exact_recv_per_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=143616 expected_recv=261631 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=143616 expected_recv=261911 expected_overflow=0 exact_recv_per_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=143616 expected_recv=261911 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=143616 expected_recv=262254 expected_overflow=0 exact_recv_per_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=143616 expected_recv=259847 expected_overflow=0 exact_recv_per_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=265600 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=143616 expected_recv=262254 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=143616 expected_recv=259847 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=143616 expected_recv=263567 expected_overflow=0 exact_recv_per_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=143616 expected_recv=259426 expected_overflow=0 exact_recv_per_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=143616 expected_recv=263567 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=143616 expected_recv=259426 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=265600 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=143616 expected_recv=259416 expected_overflow=0 exact_recv_per_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=143616 expected_recv=262169 expected_overflow=0 exact_recv_per_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=143616 expected_recv=259754 expected_overflow=0 exact_recv_per_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=143616 expected_recv=259416 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=143616 expected_recv=262169 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=143616 expected_recv=259754 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=143616 expected_recv=257107 expected_overflow=0 exact_recv_per_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=143616 expected_recv=260255 expected_overflow=0 exact_recv_per_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=143616 expected_recv=257107 actual_valid_recv=258944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=143616 expected_recv=260255 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=258944 slack_slots=28288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=143616 expected_recv=261956 expected_overflow=0 exact_recv_per_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=143616 expected_recv=261956 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=143616 expected_recv=263276 expected_overflow=0 exact_recv_per_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=143616 expected_recv=258708 expected_overflow=0 exact_recv_per_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=143616 expected_recv=263276 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=143616 expected_recv=258708 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=143616 expected_recv=259585 expected_overflow=0 exact_recv_per_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=260608 slack_slots=26624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=143616 expected_recv=259585 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=143616 expected_recv=258153 expected_overflow=0 exact_recv_per_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=143616 expected_recv=254148 expected_overflow=0 exact_recv_per_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=143616 expected_recv=258153 actual_valid_recv=260224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=143616 expected_recv=254148 actual_valid_recv=256768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=143616 expected_recv=262493 expected_overflow=0 exact_recv_per_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=260224 slack_slots=27008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=256768 slack_slots=30464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=143616 expected_recv=262493 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=143616 expected_recv=262196 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=143616 expected_recv=261618 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=143616 expected_recv=262196 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=143616 expected_recv=257857 expected_overflow=0 exact_recv_per_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=143616 expected_recv=261618 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=143616 expected_recv=257857 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=143616 expected_recv=259405 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=260352 slack_slots=26880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=143616 expected_recv=259405 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=143616 expected_recv=260348 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=143616 expected_recv=260348 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=143616 expected_recv=259880 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=143616 expected_recv=259287 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=143616 expected_recv=259880 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=143616 expected_recv=262254 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=143616 expected_recv=259287 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=143616 expected_recv=262254 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=143616 expected_recv=258364 expected_overflow=0 exact_recv_per_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=264960 slack_slots=22272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=143616 expected_recv=258364 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=259712 slack_slots=27520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=143616 expected_recv=259466 expected_overflow=0 exact_recv_per_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=143616 expected_recv=261770 expected_overflow=0 exact_recv_per_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=143616 expected_recv=259466 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=143616 expected_recv=263488 expected_overflow=0 exact_recv_per_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=143616 expected_recv=251188 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=143616 expected_recv=261770 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=143616 expected_recv=262420 expected_overflow=0 exact_recv_per_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=143616 expected_recv=263488 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=143616 expected_recv=251188 actual_valid_recv=253312 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=143616 expected_recv=262420 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=253312 slack_slots=33920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=264960 slack_slots=22272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=143616 expected_recv=262178 expected_overflow=0 exact_recv_per_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=143616 expected_recv=262178 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=143616 expected_recv=264212 expected_overflow=0 exact_recv_per_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=143616 expected_recv=264212 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=143616 expected_recv=257643 expected_overflow=0 exact_recv_per_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=143616 expected_recv=262112 expected_overflow=0 exact_recv_per_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=143616 expected_recv=259219 expected_overflow=0 exact_recv_per_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=143616 expected_recv=257643 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=143616 expected_recv=262112 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=143616 expected_recv=259219 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=259456 slack_slots=27776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[153259, 145708, 116536, 96988, 147510, 89229, 134861, 164485] avg=131072.0000 max=164485 max_rank=7 min=89229 min_rank=5 imbalance_ratio=1.254921 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[153259, 145708, 116536, 96988, 147510, 89229, 134861, 164485] avg=131072.0000 max=164485 max_rank=7 min=89229 min_rank=5 imbalance_ratio=1.254921 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[153259, 145708, 116536, 96988, 147510, 89229, 134861, 164485] avg=131072.0000 max=164485 max_rank=7 min=89229 min_rank=5 imbalance_ratio=1.254921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[153259, 145708, 116536, 96988, 147510, 89229, 134861, 164485] avg=131072.0000 max=164485 max_rank=7 min=89229 min_rank=5 imbalance_ratio=1.254921 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127548 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=32102 max_load_reduction_pct=0.195167 moved_to_replicas=164027 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127548 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=32102 max_load_reduction_pct=0.195167 moved_to_replicas=164027 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=143616 expected_recv=259602 expected_overflow=0 exact_recv_per_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127548 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=32102 max_load_reduction_pct=0.195167 moved_to_replicas=164027 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[153259, 145708, 116536, 96988, 147510, 89229, 134861, 164485] avg=131072.0000 max=164485 max_rank=7 min=89229 min_rank=5 imbalance_ratio=1.254921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[153259, 145708, 116536, 96988, 147510, 89229, 134861, 164485] avg=131072.0000 max=164485 max_rank=7 min=89229 min_rank=5 imbalance_ratio=1.254921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127548 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=32102 max_load_reduction_pct=0.195167 moved_to_replicas=164027 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=143616 expected_recv=259602 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[153259, 145708, 116536, 96988, 147510, 89229, 134861, 164485] avg=131072.0000 max=164485 max_rank=7 min=89229 min_rank=5 imbalance_ratio=1.254921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[92464, 159837, 88816, 177760, 144130, 102572, 162899, 120098] avg=131072.0000 max=177760 max_rank=3 min=88816 min_rank=2 imbalance_ratio=1.356201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[92464, 159837, 88816, 177760, 144130, 102572, 162899, 120098] avg=131072.0000 max=177760 max_rank=3 min=88816 min_rank=2 imbalance_ratio=1.356201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127548 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=32102 max_load_reduction_pct=0.195167 moved_to_replicas=164027 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127548 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=32102 max_load_reduction_pct=0.195167 moved_to_replicas=164027 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[92464, 159837, 88816, 177760, 144130, 102572, 162899, 120098] avg=131072.0000 max=177760 max_rank=3 min=88816 min_rank=2 imbalance_ratio=1.356201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[92464, 159837, 88816, 177760, 144130, 102572, 162899, 120098] avg=131072.0000 max=177760 max_rank=3 min=88816 min_rank=2 imbalance_ratio=1.356201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127548 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=32102 max_load_reduction_pct=0.195167 moved_to_replicas=164027 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=45376 max_load_reduction_pct=0.255266 moved_to_replicas=231232 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=45376 max_load_reduction_pct=0.255266 moved_to_replicas=231232 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[153259, 145708, 116536, 96988, 147510, 89229, 134861, 164485] avg=131072.0000 max=164485 max_rank=7 min=89229 min_rank=5 imbalance_ratio=1.254921 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=45376 max_load_reduction_pct=0.255266 moved_to_replicas=231232 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=45376 max_load_reduction_pct=0.255266 moved_to_replicas=231232 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[92464, 159837, 88816, 177760, 144130, 102572, 162899, 120098] avg=131072.0000 max=177760 max_rank=3 min=88816 min_rank=2 imbalance_ratio=1.356201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[92464, 159837, 88816, 177760, 144130, 102572, 162899, 120098] avg=131072.0000 max=177760 max_rank=3 min=88816 min_rank=2 imbalance_ratio=1.356201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132383, 129861, 127548, 132382, 129253, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127548 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=32102 max_load_reduction_pct=0.195167 moved_to_replicas=164027 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[145178, 172540, 133576, 189415, 130433, 144400, 37970, 95064] avg=131072.0000 max=189415 max_rank=3 min=37970 min_rank=6 imbalance_ratio=1.445122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[92464, 159837, 88816, 177760, 144130, 102572, 162899, 120098] avg=131072.0000 max=177760 max_rank=3 min=88816 min_rank=2 imbalance_ratio=1.356201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[145178, 172540, 133576, 189415, 130433, 144400, 37970, 95064] avg=131072.0000 max=189415 max_rank=3 min=37970 min_rank=6 imbalance_ratio=1.445122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=45376 max_load_reduction_pct=0.255266 moved_to_replicas=231232 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[145178, 172540, 133576, 189415, 130433, 144400, 37970, 95064] avg=131072.0000 max=189415 max_rank=3 min=37970 min_rank=6 imbalance_ratio=1.445122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=45376 max_load_reduction_pct=0.255266 moved_to_replicas=231232 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[145178, 172540, 133576, 189415, 130433, 144400, 37970, 95064] avg=131072.0000 max=189415 max_rank=3 min=37970 min_rank=6 imbalance_ratio=1.445122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] avg=131072.0000 max=132385 max_rank=3 min=127758 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=57030 max_load_reduction_pct=0.301085 moved_to_replicas=245428 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] avg=131072.0000 max=132385 max_rank=3 min=127758 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=57030 max_load_reduction_pct=0.301085 moved_to_replicas=245428 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=45376 max_load_reduction_pct=0.255266 moved_to_replicas=231232 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] avg=131072.0000 max=132385 max_rank=3 min=127758 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=57030 max_load_reduction_pct=0.301085 moved_to_replicas=245428 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] avg=131072.0000 max=132385 max_rank=3 min=127758 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=57030 max_load_reduction_pct=0.301085 moved_to_replicas=245428 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[92464, 159837, 88816, 177760, 144130, 102572, 162899, 120098] avg=131072.0000 max=177760 max_rank=3 min=88816 min_rank=2 imbalance_ratio=1.356201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[145178, 172540, 133576, 189415, 130433, 144400, 37970, 95064] avg=131072.0000 max=189415 max_rank=3 min=37970 min_rank=6 imbalance_ratio=1.445122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[145178, 172540, 133576, 189415, 130433, 144400, 37970, 95064] avg=131072.0000 max=189415 max_rank=3 min=37970 min_rank=6 imbalance_ratio=1.445122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[163277, 157223, 163737, 163856, 98298, 131240, 65631, 105314] avg=131072.0000 max=163856 max_rank=3 min=65631 min_rank=6 imbalance_ratio=1.250122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[163277, 157223, 163737, 163856, 98298, 131240, 65631, 105314] avg=131072.0000 max=163856 max_rank=3 min=65631 min_rank=6 imbalance_ratio=1.250122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[145178, 172540, 133576, 189415, 130433, 144400, 37970, 95064] avg=131072.0000 max=189415 max_rank=3 min=37970 min_rank=6 imbalance_ratio=1.445122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[132383, 132384, 121895, 132382, 132382, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=45376 max_load_reduction_pct=0.255266 moved_to_replicas=231232 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[163277, 157223, 163737, 163856, 98298, 131240, 65631, 105314] avg=131072.0000 max=163856 max_rank=3 min=65631 min_rank=6 imbalance_ratio=1.250122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] avg=131072.0000 max=132385 max_rank=3 min=127758 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=57030 max_load_reduction_pct=0.301085 moved_to_replicas=245428 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] avg=131072.0000 max=132385 max_rank=3 min=127758 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=57030 max_load_reduction_pct=0.301085 moved_to_replicas=245428 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[163277, 157223, 163737, 163856, 98298, 131240, 65631, 105314] avg=131072.0000 max=163856 max_rank=3 min=65631 min_rank=6 imbalance_ratio=1.250122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] avg=131072.0000 max=132383 max_rank=0 min=128458 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31473 max_load_reduction_pct=0.192077 moved_to_replicas=238141 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] avg=131072.0000 max=132383 max_rank=0 min=128458 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31473 max_load_reduction_pct=0.192077 moved_to_replicas=238141 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] avg=131072.0000 max=132385 max_rank=3 min=127758 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=57030 max_load_reduction_pct=0.301085 moved_to_replicas=245428 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] avg=131072.0000 max=132383 max_rank=0 min=128458 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31473 max_load_reduction_pct=0.192077 moved_to_replicas=238141 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] avg=131072.0000 max=132383 max_rank=0 min=128458 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31473 max_load_reduction_pct=0.192077 moved_to_replicas=238141 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[145178, 172540, 133576, 189415, 130433, 144400, 37970, 95064] avg=131072.0000 max=189415 max_rank=3 min=37970 min_rank=6 imbalance_ratio=1.445122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[163277, 157223, 163737, 163856, 98298, 131240, 65631, 105314] avg=131072.0000 max=163856 max_rank=3 min=65631 min_rank=6 imbalance_ratio=1.250122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[163277, 157223, 163737, 163856, 98298, 131240, 65631, 105314] avg=131072.0000 max=163856 max_rank=3 min=65631 min_rank=6 imbalance_ratio=1.250122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[95664, 98220, 131350, 131933, 131032, 102922, 150181, 207274] avg=131072.0000 max=207274 max_rank=7 min=95664 min_rank=0 imbalance_ratio=1.581375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[95664, 98220, 131350, 131933, 131032, 102922, 150181, 207274] avg=131072.0000 max=207274 max_rank=7 min=95664 min_rank=0 imbalance_ratio=1.581375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[95664, 98220, 131350, 131933, 131032, 102922, 150181, 207274] avg=131072.0000 max=207274 max_rank=7 min=95664 min_rank=0 imbalance_ratio=1.581375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[163277, 157223, 163737, 163856, 98298, 131240, 65631, 105314] avg=131072.0000 max=163856 max_rank=3 min=65631 min_rank=6 imbalance_ratio=1.250122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[132384, 132382, 132383, 132385, 130433, 132383, 127758, 128468] avg=131072.0000 max=132385 max_rank=3 min=127758 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=57030 max_load_reduction_pct=0.301085 moved_to_replicas=245428 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[95664, 98220, 131350, 131933, 131032, 102922, 150181, 207274] avg=131072.0000 max=207274 max_rank=7 min=95664 min_rank=0 imbalance_ratio=1.581375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] avg=131072.0000 max=132383 max_rank=0 min=128458 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31473 max_load_reduction_pct=0.192077 moved_to_replicas=238141 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] avg=131072.0000 max=132383 max_rank=0 min=128458 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31473 max_load_reduction_pct=0.192077 moved_to_replicas=238141 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] avg=131072.0000 max=132383 max_rank=7 min=128428 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=74891 max_load_reduction_pct=0.361314 moved_to_replicas=186344 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] avg=131072.0000 max=132383 max_rank=7 min=128428 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=74891 max_load_reduction_pct=0.361314 moved_to_replicas=186344 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] avg=131072.0000 max=132383 max_rank=7 min=128428 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=74891 max_load_reduction_pct=0.361314 moved_to_replicas=186344 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] avg=131072.0000 max=132383 max_rank=0 min=128458 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31473 max_load_reduction_pct=0.192077 moved_to_replicas=238141 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] avg=131072.0000 max=132383 max_rank=7 min=128428 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=74891 max_load_reduction_pct=0.361314 moved_to_replicas=186344 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[163277, 157223, 163737, 163856, 98298, 131240, 65631, 105314] avg=131072.0000 max=163856 max_rank=3 min=65631 min_rank=6 imbalance_ratio=1.250122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[95664, 98220, 131350, 131933, 131032, 102922, 150181, 207274] avg=131072.0000 max=207274 max_rank=7 min=95664 min_rank=0 imbalance_ratio=1.581375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[145637, 98375, 171502, 107073, 130997, 230720, 98233, 66039] avg=131072.0000 max=230720 max_rank=5 min=66039 min_rank=7 imbalance_ratio=1.760254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[95664, 98220, 131350, 131933, 131032, 102922, 150181, 207274] avg=131072.0000 max=207274 max_rank=7 min=95664 min_rank=0 imbalance_ratio=1.581375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[145637, 98375, 171502, 107073, 130997, 230720, 98233, 66039] avg=131072.0000 max=230720 max_rank=5 min=66039 min_rank=7 imbalance_ratio=1.760254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[145637, 98375, 171502, 107073, 130997, 230720, 98233, 66039] avg=131072.0000 max=230720 max_rank=5 min=66039 min_rank=7 imbalance_ratio=1.760254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[95664, 98220, 131350, 131933, 131032, 102922, 150181, 207274] avg=131072.0000 max=207274 max_rank=7 min=95664 min_rank=0 imbalance_ratio=1.581375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132381, 132383, 132383, 129192, 131240, 128458, 130156] avg=131072.0000 max=132383 max_rank=0 min=128458 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31473 max_load_reduction_pct=0.192077 moved_to_replicas=238141 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[145637, 98375, 171502, 107073, 130997, 230720, 98233, 66039] avg=131072.0000 max=230720 max_rank=5 min=66039 min_rank=7 imbalance_ratio=1.760254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] avg=131072.0000 max=132383 max_rank=5 min=126685 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=98337 max_load_reduction_pct=0.426218 moved_to_replicas=306177 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] avg=131072.0000 max=132383 max_rank=7 min=128428 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=74891 max_load_reduction_pct=0.361314 moved_to_replicas=186344 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] avg=131072.0000 max=132383 max_rank=5 min=126685 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=98337 max_load_reduction_pct=0.426218 moved_to_replicas=306177 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] avg=131072.0000 max=132383 max_rank=7 min=128428 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=74891 max_load_reduction_pct=0.361314 moved_to_replicas=186344 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] avg=131072.0000 max=132383 max_rank=5 min=126685 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=98337 max_load_reduction_pct=0.426218 moved_to_replicas=306177 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] avg=131072.0000 max=132383 max_rank=7 min=128428 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=74891 max_load_reduction_pct=0.361314 moved_to_replicas=186344 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] avg=131072.0000 max=132383 max_rank=5 min=126685 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=98337 max_load_reduction_pct=0.426218 moved_to_replicas=306177 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[95664, 98220, 131350, 131933, 131032, 102922, 150181, 207274] avg=131072.0000 max=207274 max_rank=7 min=95664 min_rank=0 imbalance_ratio=1.581375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65672, 98786, 130861, 220568, 109141, 253954, 104091, 65503] avg=131072.0000 max=253954 max_rank=5 min=65503 min_rank=7 imbalance_ratio=1.937515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[145637, 98375, 171502, 107073, 130997, 230720, 98233, 66039] avg=131072.0000 max=230720 max_rank=5 min=66039 min_rank=7 imbalance_ratio=1.760254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65672, 98786, 130861, 220568, 109141, 253954, 104091, 65503] avg=131072.0000 max=253954 max_rank=5 min=65503 min_rank=7 imbalance_ratio=1.937515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65672, 98786, 130861, 220568, 109141, 253954, 104091, 65503] avg=131072.0000 max=253954 max_rank=5 min=65503 min_rank=7 imbalance_ratio=1.937515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[145637, 98375, 171502, 107073, 130997, 230720, 98233, 66039] avg=131072.0000 max=230720 max_rank=5 min=66039 min_rank=7 imbalance_ratio=1.760254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[145637, 98375, 171502, 107073, 130997, 230720, 98233, 66039] avg=131072.0000 max=230720 max_rank=5 min=66039 min_rank=7 imbalance_ratio=1.760254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[128428, 130984, 131350, 131933, 131032, 130084, 132382, 132383] avg=131072.0000 max=132383 max_rank=7 min=128428 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=74891 max_load_reduction_pct=0.361314 moved_to_replicas=186344 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65672, 98786, 130861, 220568, 109141, 253954, 104091, 65503] avg=131072.0000 max=253954 max_rank=5 min=65503 min_rank=7 imbalance_ratio=1.937515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] avg=131072.0000 max=132384 max_rank=3 min=125603 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=121570 max_load_reduction_pct=0.478709 moved_to_replicas=418174 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] avg=131072.0000 max=132383 max_rank=5 min=126685 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=98337 max_load_reduction_pct=0.426218 moved_to_replicas=306177 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] avg=131072.0000 max=132384 max_rank=3 min=125603 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=121570 max_load_reduction_pct=0.478709 moved_to_replicas=418174 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] avg=131072.0000 max=132384 max_rank=3 min=125603 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=121570 max_load_reduction_pct=0.478709 moved_to_replicas=418174 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] avg=131072.0000 max=132383 max_rank=5 min=126685 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=98337 max_load_reduction_pct=0.426218 moved_to_replicas=306177 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] avg=131072.0000 max=132383 max_rank=5 min=126685 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=98337 max_load_reduction_pct=0.426218 moved_to_replicas=306177 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] avg=131072.0000 max=132384 max_rank=3 min=125603 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=121570 max_load_reduction_pct=0.478709 moved_to_replicas=418174 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[145637, 98375, 171502, 107073, 130997, 230720, 98233, 66039] avg=131072.0000 max=230720 max_rank=5 min=66039 min_rank=7 imbalance_ratio=1.760254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[98124, 88726, 198980, 98323, 98381, 138643, 96664, 230735] avg=131072.0000 max=230735 max_rank=7 min=88726 min_rank=1 imbalance_ratio=1.760368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65672, 98786, 130861, 220568, 109141, 253954, 104091, 65503] avg=131072.0000 max=253954 max_rank=5 min=65503 min_rank=7 imbalance_ratio=1.937515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[98124, 88726, 198980, 98323, 98381, 138643, 96664, 230735] avg=131072.0000 max=230735 max_rank=7 min=88726 min_rank=1 imbalance_ratio=1.760368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[98124, 88726, 198980, 98323, 98381, 138643, 96664, 230735] avg=131072.0000 max=230735 max_rank=7 min=88726 min_rank=1 imbalance_ratio=1.760368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65672, 98786, 130861, 220568, 109141, 253954, 104091, 65503] avg=131072.0000 max=253954 max_rank=5 min=65503 min_rank=7 imbalance_ratio=1.937515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[98124, 88726, 198980, 98323, 98381, 138643, 96664, 230735] avg=131072.0000 max=230735 max_rank=7 min=88726 min_rank=1 imbalance_ratio=1.760368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65672, 98786, 130861, 220568, 109141, 253954, 104091, 65503] avg=131072.0000 max=253954 max_rank=5 min=65503 min_rank=7 imbalance_ratio=1.937515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[132381, 131138, 132382, 126685, 130997, 132383, 130998, 131612] avg=131072.0000 max=132383 max_rank=5 min=126685 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=98337 max_load_reduction_pct=0.426218 moved_to_replicas=306177 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] avg=131072.0000 max=132383 max_rank=2 min=128822 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=98352 max_load_reduction_pct=0.426255 moved_to_replicas=340551 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] avg=131072.0000 max=132384 max_rank=3 min=125603 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=121570 max_load_reduction_pct=0.478709 moved_to_replicas=418174 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] avg=131072.0000 max=132383 max_rank=2 min=128822 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=98352 max_load_reduction_pct=0.426255 moved_to_replicas=340551 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] avg=131072.0000 max=132383 max_rank=2 min=128822 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=98352 max_load_reduction_pct=0.426255 moved_to_replicas=340551 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] avg=131072.0000 max=132383 max_rank=2 min=128822 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=98352 max_load_reduction_pct=0.426255 moved_to_replicas=340551 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] avg=131072.0000 max=132384 max_rank=3 min=125603 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=121570 max_load_reduction_pct=0.478709 moved_to_replicas=418174 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] avg=131072.0000 max=132384 max_rank=3 min=125603 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=121570 max_load_reduction_pct=0.478709 moved_to_replicas=418174 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[98378, 130981, 65751, 163606, 98514, 98106, 196605, 196635] avg=131072.0000 max=196635 max_rank=7 min=65751 min_rank=2 imbalance_ratio=1.500206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65672, 98786, 130861, 220568, 109141, 253954, 104091, 65503] avg=131072.0000 max=253954 max_rank=5 min=65503 min_rank=7 imbalance_ratio=1.937515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[98124, 88726, 198980, 98323, 98381, 138643, 96664, 230735] avg=131072.0000 max=230735 max_rank=7 min=88726 min_rank=1 imbalance_ratio=1.760368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[98378, 130981, 65751, 163606, 98514, 98106, 196605, 196635] avg=131072.0000 max=196635 max_rank=7 min=65751 min_rank=2 imbalance_ratio=1.500206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[98378, 130981, 65751, 163606, 98514, 98106, 196605, 196635] avg=131072.0000 max=196635 max_rank=7 min=65751 min_rank=2 imbalance_ratio=1.500206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[98378, 130981, 65751, 163606, 98514, 98106, 196605, 196635] avg=131072.0000 max=196635 max_rank=7 min=65751 min_rank=2 imbalance_ratio=1.500206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129593 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64252 max_load_reduction_pct=0.326758 moved_to_replicas=319599 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[98124, 88726, 198980, 98323, 98381, 138643, 96664, 230735] avg=131072.0000 max=230735 max_rank=7 min=88726 min_rank=1 imbalance_ratio=1.760368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[98124, 88726, 198980, 98323, 98381, 138643, 96664, 230735] avg=131072.0000 max=230735 max_rank=7 min=88726 min_rank=1 imbalance_ratio=1.760368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[125603, 131549, 130861, 132384, 132384, 132382, 132381, 131032] avg=131072.0000 max=132384 max_rank=3 min=125603 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=121570 max_load_reduction_pct=0.478709 moved_to_replicas=418174 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] avg=131072.0000 max=132383 max_rank=2 min=128822 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=98352 max_load_reduction_pct=0.426255 moved_to_replicas=340551 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129593 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64252 max_load_reduction_pct=0.326758 moved_to_replicas=319599 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129593 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64252 max_load_reduction_pct=0.326758 moved_to_replicas=319599 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129593 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64252 max_load_reduction_pct=0.326758 moved_to_replicas=319599 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] avg=131072.0000 max=132383 max_rank=2 min=128822 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=98352 max_load_reduction_pct=0.426255 moved_to_replicas=340551 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] avg=131072.0000 max=132383 max_rank=2 min=128822 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=98352 max_load_reduction_pct=0.426255 moved_to_replicas=340551 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[128863, 71308, 159721, 98457, 65629, 163811, 131070, 229717] avg=131072.0000 max=229717 max_rank=7 min=65629 min_rank=4 imbalance_ratio=1.752602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[98124, 88726, 198980, 98323, 98381, 138643, 96664, 230735] avg=131072.0000 max=230735 max_rank=7 min=88726 min_rank=1 imbalance_ratio=1.760368 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[98378, 130981, 65751, 163606, 98514, 98106, 196605, 196635] avg=131072.0000 max=196635 max_rank=7 min=65751 min_rank=2 imbalance_ratio=1.500206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[128863, 71308, 159721, 98457, 65629, 163811, 131070, 229717] avg=131072.0000 max=229717 max_rank=7 min=65629 min_rank=4 imbalance_ratio=1.752602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[128863, 71308, 159721, 98457, 65629, 163811, 131070, 229717] avg=131072.0000 max=229717 max_rank=7 min=65629 min_rank=4 imbalance_ratio=1.752602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[128863, 71308, 159721, 98457, 65629, 163811, 131070, 229717] avg=131072.0000 max=229717 max_rank=7 min=65629 min_rank=4 imbalance_ratio=1.752602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] avg=131072.0000 max=132384 max_rank=2 min=128863 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97333 max_load_reduction_pct=0.423708 moved_to_replicas=312318 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[130888, 128822, 132383, 131145, 131146, 132381, 129428, 132383] avg=131072.0000 max=132383 max_rank=2 min=128822 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=98352 max_load_reduction_pct=0.426255 moved_to_replicas=340551 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[98378, 130981, 65751, 163606, 98514, 98106, 196605, 196635] avg=131072.0000 max=196635 max_rank=7 min=65751 min_rank=2 imbalance_ratio=1.500206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] avg=131072.0000 max=132384 max_rank=2 min=128863 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97333 max_load_reduction_pct=0.423708 moved_to_replicas=312318 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129593 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64252 max_load_reduction_pct=0.326758 moved_to_replicas=319599 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] avg=131072.0000 max=132384 max_rank=2 min=128863 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97333 max_load_reduction_pct=0.423708 moved_to_replicas=312318 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[98378, 130981, 65751, 163606, 98514, 98106, 196605, 196635] avg=131072.0000 max=196635 max_rank=7 min=65751 min_rank=2 imbalance_ratio=1.500206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] avg=131072.0000 max=132384 max_rank=2 min=128863 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97333 max_load_reduction_pct=0.423708 moved_to_replicas=312318 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129593 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64252 max_load_reduction_pct=0.326758 moved_to_replicas=319599 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[34327, 196612, 32991, 195888, 98504, 260950, 65507, 163797] avg=131072.0000 max=260950 max_rank=5 min=32991 min_rank=2 imbalance_ratio=1.990891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129593 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64252 max_load_reduction_pct=0.326758 moved_to_replicas=319599 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[98378, 130981, 65751, 163606, 98514, 98106, 196605, 196635] avg=131072.0000 max=196635 max_rank=7 min=65751 min_rank=2 imbalance_ratio=1.500206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[34327, 196612, 32991, 195888, 98504, 260950, 65507, 163797] avg=131072.0000 max=260950 max_rank=5 min=32991 min_rank=2 imbalance_ratio=1.990891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[128863, 71308, 159721, 98457, 65629, 163811, 131070, 229717] avg=131072.0000 max=229717 max_rank=7 min=65629 min_rank=4 imbalance_ratio=1.752602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[34327, 196612, 32991, 195888, 98504, 260950, 65507, 163797] avg=131072.0000 max=260950 max_rank=5 min=32991 min_rank=2 imbalance_ratio=1.990891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[34327, 196612, 32991, 195888, 98504, 260950, 65507, 163797] avg=131072.0000 max=260950 max_rank=5 min=32991 min_rank=2 imbalance_ratio=1.990891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] avg=131072.0000 max=132383 max_rank=1 min=127498 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=128567 max_load_reduction_pct=0.492688 moved_to_replicas=574369 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[131142, 130981, 129739, 132383, 129972, 129593, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129593 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64252 max_load_reduction_pct=0.326758 moved_to_replicas=319599 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] avg=131072.0000 max=132383 max_rank=1 min=127498 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=128567 max_load_reduction_pct=0.492688 moved_to_replicas=574369 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[128863, 71308, 159721, 98457, 65629, 163811, 131070, 229717] avg=131072.0000 max=229717 max_rank=7 min=65629 min_rank=4 imbalance_ratio=1.752602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] avg=131072.0000 max=132384 max_rank=2 min=128863 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97333 max_load_reduction_pct=0.423708 moved_to_replicas=312318 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] avg=131072.0000 max=132383 max_rank=1 min=127498 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=128567 max_load_reduction_pct=0.492688 moved_to_replicas=574369 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] avg=131072.0000 max=132383 max_rank=1 min=127498 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=128567 max_load_reduction_pct=0.492688 moved_to_replicas=574369 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[128863, 71308, 159721, 98457, 65629, 163811, 131070, 229717] avg=131072.0000 max=229717 max_rank=7 min=65629 min_rank=4 imbalance_ratio=1.752602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[164009, 98485, 65239, 163645, 131126, 164978, 163700, 97394] avg=131072.0000 max=164978 max_rank=5 min=65239 min_rank=2 imbalance_ratio=1.258682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] avg=131072.0000 max=132384 max_rank=2 min=128863 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97333 max_load_reduction_pct=0.423708 moved_to_replicas=312318 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] avg=131072.0000 max=132384 max_rank=2 min=128863 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97333 max_load_reduction_pct=0.423708 moved_to_replicas=312318 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[128863, 71308, 159721, 98457, 65629, 163811, 131070, 229717] avg=131072.0000 max=229717 max_rank=7 min=65629 min_rank=4 imbalance_ratio=1.752602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[164009, 98485, 65239, 163645, 131126, 164978, 163700, 97394] avg=131072.0000 max=164978 max_rank=5 min=65239 min_rank=2 imbalance_ratio=1.258682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[164009, 98485, 65239, 163645, 131126, 164978, 163700, 97394] avg=131072.0000 max=164978 max_rank=5 min=65239 min_rank=2 imbalance_ratio=1.258682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[34327, 196612, 32991, 195888, 98504, 260950, 65507, 163797] avg=131072.0000 max=260950 max_rank=5 min=32991 min_rank=2 imbalance_ratio=1.990891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[164009, 98485, 65239, 163645, 131126, 164978, 163700, 97394] avg=131072.0000 max=164978 max_rank=5 min=65239 min_rank=2 imbalance_ratio=1.258682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] avg=131072.0000 max=132383 max_rank=0 min=129020 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32595 max_load_reduction_pct=0.197572 moved_to_replicas=252988 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[128863, 131408, 132384, 129885, 130200, 132383, 131070, 132383] avg=131072.0000 max=132384 max_rank=2 min=128863 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=97333 max_load_reduction_pct=0.423708 moved_to_replicas=312318 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[34327, 196612, 32991, 195888, 98504, 260950, 65507, 163797] avg=131072.0000 max=260950 max_rank=5 min=32991 min_rank=2 imbalance_ratio=1.990891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] avg=131072.0000 max=132383 max_rank=0 min=129020 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32595 max_load_reduction_pct=0.197572 moved_to_replicas=252988 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] avg=131072.0000 max=132383 max_rank=0 min=129020 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32595 max_load_reduction_pct=0.197572 moved_to_replicas=252988 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] avg=131072.0000 max=132383 max_rank=1 min=127498 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=128567 max_load_reduction_pct=0.492688 moved_to_replicas=574369 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] avg=131072.0000 max=132383 max_rank=0 min=129020 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32595 max_load_reduction_pct=0.197572 moved_to_replicas=252988 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[34327, 196612, 32991, 195888, 98504, 260950, 65507, 163797] avg=131072.0000 max=260950 max_rank=5 min=32991 min_rank=2 imbalance_ratio=1.990891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[196727, 196589, 65563, 163721, 98300, 65750, 131116, 130810] avg=131072.0000 max=196727 max_rank=0 min=65563 min_rank=2 imbalance_ratio=1.500908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] avg=131072.0000 max=132383 max_rank=1 min=127498 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=128567 max_load_reduction_pct=0.492688 moved_to_replicas=574369 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] avg=131072.0000 max=132383 max_rank=1 min=127498 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=128567 max_load_reduction_pct=0.492688 moved_to_replicas=574369 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[34327, 196612, 32991, 195888, 98504, 260950, 65507, 163797] avg=131072.0000 max=260950 max_rank=5 min=32991 min_rank=2 imbalance_ratio=1.990891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[196727, 196589, 65563, 163721, 98300, 65750, 131116, 130810] avg=131072.0000 max=196727 max_rank=0 min=65563 min_rank=2 imbalance_ratio=1.500908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[196727, 196589, 65563, 163721, 98300, 65750, 131116, 130810] avg=131072.0000 max=196727 max_rank=0 min=65563 min_rank=2 imbalance_ratio=1.500908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[164009, 98485, 65239, 163645, 131126, 164978, 163700, 97394] avg=131072.0000 max=164978 max_rank=5 min=65239 min_rank=2 imbalance_ratio=1.258682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[196727, 196589, 65563, 163721, 98300, 65750, 131116, 130810] avg=131072.0000 max=196727 max_rank=0 min=65563 min_rank=2 imbalance_ratio=1.500908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] avg=131072.0000 max=132383 max_rank=0 min=129664 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64344 max_load_reduction_pct=0.327073 moved_to_replicas=319955 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] avg=131072.0000 max=132383 max_rank=0 min=129664 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64344 max_load_reduction_pct=0.327073 moved_to_replicas=319955 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[131268, 132383, 127498, 132383, 129245, 132383, 131033, 132383] avg=131072.0000 max=132383 max_rank=1 min=127498 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=128567 max_load_reduction_pct=0.492688 moved_to_replicas=574369 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] avg=131072.0000 max=132383 max_rank=0 min=129664 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64344 max_load_reduction_pct=0.327073 moved_to_replicas=319955 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[164009, 98485, 65239, 163645, 131126, 164978, 163700, 97394] avg=131072.0000 max=164978 max_rank=5 min=65239 min_rank=2 imbalance_ratio=1.258682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] avg=131072.0000 max=132383 max_rank=0 min=129020 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32595 max_load_reduction_pct=0.197572 moved_to_replicas=252988 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] avg=131072.0000 max=132383 max_rank=0 min=129664 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64344 max_load_reduction_pct=0.327073 moved_to_replicas=319955 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[164009, 98485, 65239, 163645, 131126, 164978, 163700, 97394] avg=131072.0000 max=164978 max_rank=5 min=65239 min_rank=2 imbalance_ratio=1.258682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[65628, 168352, 124585, 165453, 98255, 196551, 163926, 65826] avg=131072.0000 max=196551 max_rank=5 min=65628 min_rank=0 imbalance_ratio=1.499565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] avg=131072.0000 max=132383 max_rank=0 min=129020 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32595 max_load_reduction_pct=0.197572 moved_to_replicas=252988 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[65628, 168352, 124585, 165453, 98255, 196551, 163926, 65826] avg=131072.0000 max=196551 max_rank=5 min=65628 min_rank=0 imbalance_ratio=1.499565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] avg=131072.0000 max=132383 max_rank=0 min=129020 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32595 max_load_reduction_pct=0.197572 moved_to_replicas=252988 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[65628, 168352, 124585, 165453, 98255, 196551, 163926, 65826] avg=131072.0000 max=196551 max_rank=5 min=65628 min_rank=0 imbalance_ratio=1.499565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[164009, 98485, 65239, 163645, 131126, 164978, 163700, 97394] avg=131072.0000 max=164978 max_rank=5 min=65239 min_rank=2 imbalance_ratio=1.258682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] avg=131072.0000 max=132383 max_rank=1 min=125511 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64168 max_load_reduction_pct=0.326470 moved_to_replicas=331102 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[196727, 196589, 65563, 163721, 98300, 65750, 131116, 130810] avg=131072.0000 max=196727 max_rank=0 min=65563 min_rank=2 imbalance_ratio=1.500908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[65628, 168352, 124585, 165453, 98255, 196551, 163926, 65826] avg=131072.0000 max=196551 max_rank=5 min=65628 min_rank=0 imbalance_ratio=1.499565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] avg=131072.0000 max=132383 max_rank=1 min=125511 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64168 max_load_reduction_pct=0.326470 moved_to_replicas=331102 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] avg=131072.0000 max=132383 max_rank=1 min=125511 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64168 max_load_reduction_pct=0.326470 moved_to_replicas=331102 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[132383, 129747, 129151, 132383, 131126, 132383, 132383, 129020] avg=131072.0000 max=132383 max_rank=0 min=129020 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32595 max_load_reduction_pct=0.197572 moved_to_replicas=252988 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[196727, 196589, 65563, 163721, 98300, 65750, 131116, 130810] avg=131072.0000 max=196727 max_rank=0 min=65563 min_rank=2 imbalance_ratio=1.500908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] avg=131072.0000 max=132383 max_rank=0 min=129664 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64344 max_load_reduction_pct=0.327073 moved_to_replicas=319955 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] avg=131072.0000 max=132383 max_rank=1 min=125511 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64168 max_load_reduction_pct=0.326470 moved_to_replicas=331102 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[131767, 98514, 131080, 98132, 131230, 162962, 229341, 65550] avg=131072.0000 max=229341 max_rank=6 min=65550 min_rank=7 imbalance_ratio=1.749733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[196727, 196589, 65563, 163721, 98300, 65750, 131116, 130810] avg=131072.0000 max=196727 max_rank=0 min=65563 min_rank=2 imbalance_ratio=1.500908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[131767, 98514, 131080, 98132, 131230, 162962, 229341, 65550] avg=131072.0000 max=229341 max_rank=6 min=65550 min_rank=7 imbalance_ratio=1.749733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] avg=131072.0000 max=132383 max_rank=0 min=129664 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64344 max_load_reduction_pct=0.327073 moved_to_replicas=319955 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] avg=131072.0000 max=132383 max_rank=5 min=129093 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96958 max_load_reduction_pct=0.422768 moved_to_replicas=255371 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[131767, 98514, 131080, 98132, 131230, 162962, 229341, 65550] avg=131072.0000 max=229341 max_rank=6 min=65550 min_rank=7 imbalance_ratio=1.749733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] avg=131072.0000 max=132383 max_rank=0 min=129664 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64344 max_load_reduction_pct=0.327073 moved_to_replicas=319955 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[196727, 196589, 65563, 163721, 98300, 65750, 131116, 130810] avg=131072.0000 max=196727 max_rank=0 min=65563 min_rank=2 imbalance_ratio=1.500908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[131767, 98514, 131080, 98132, 131230, 162962, 229341, 65550] avg=131072.0000 max=229341 max_rank=6 min=65550 min_rank=7 imbalance_ratio=1.749733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[65628, 168352, 124585, 165453, 98255, 196551, 163926, 65826] avg=131072.0000 max=196551 max_rank=5 min=65628 min_rank=0 imbalance_ratio=1.499565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] avg=131072.0000 max=132383 max_rank=5 min=129093 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96958 max_load_reduction_pct=0.422768 moved_to_replicas=255371 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] avg=131072.0000 max=132383 max_rank=5 min=129093 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96958 max_load_reduction_pct=0.422768 moved_to_replicas=255371 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129664, 132383, 129743, 130094, 131116, 130810] avg=131072.0000 max=132383 max_rank=0 min=129664 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64344 max_load_reduction_pct=0.327073 moved_to_replicas=319955 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] avg=131072.0000 max=132383 max_rank=5 min=129093 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96958 max_load_reduction_pct=0.422768 moved_to_replicas=255371 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[65628, 168352, 124585, 165453, 98255, 196551, 163926, 65826] avg=131072.0000 max=196551 max_rank=5 min=65628 min_rank=0 imbalance_ratio=1.499565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[33216, 163836, 65581, 132325, 96599, 229177, 131093, 196749] avg=131072.0000 max=229177 max_rank=5 min=33216 min_rank=0 imbalance_ratio=1.748482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] avg=131072.0000 max=132383 max_rank=1 min=125511 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64168 max_load_reduction_pct=0.326470 moved_to_replicas=331102 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[65628, 168352, 124585, 165453, 98255, 196551, 163926, 65826] avg=131072.0000 max=196551 max_rank=5 min=65628 min_rank=0 imbalance_ratio=1.499565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[33216, 163836, 65581, 132325, 96599, 229177, 131093, 196749] avg=131072.0000 max=229177 max_rank=5 min=33216 min_rank=0 imbalance_ratio=1.748482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] avg=131072.0000 max=132383 max_rank=1 min=125511 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64168 max_load_reduction_pct=0.326470 moved_to_replicas=331102 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] avg=131072.0000 max=132383 max_rank=1 min=128861 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96794 max_load_reduction_pct=0.422355 moved_to_replicas=385322 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[33216, 163836, 65581, 132325, 96599, 229177, 131093, 196749] avg=131072.0000 max=229177 max_rank=5 min=33216 min_rank=0 imbalance_ratio=1.748482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[65628, 168352, 124585, 165453, 98255, 196551, 163926, 65826] avg=131072.0000 max=196551 max_rank=5 min=65628 min_rank=0 imbalance_ratio=1.499565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[33216, 163836, 65581, 132325, 96599, 229177, 131093, 196749] avg=131072.0000 max=229177 max_rank=5 min=33216 min_rank=0 imbalance_ratio=1.748482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] avg=131072.0000 max=132383 max_rank=1 min=125511 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64168 max_load_reduction_pct=0.326470 moved_to_replicas=331102 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[131767, 98514, 131080, 98132, 131230, 162962, 229341, 65550] avg=131072.0000 max=229341 max_rank=6 min=65550 min_rank=7 imbalance_ratio=1.749733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] avg=131072.0000 max=132383 max_rank=1 min=128861 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96794 max_load_reduction_pct=0.422355 moved_to_replicas=385322 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] avg=131072.0000 max=132383 max_rank=1 min=128861 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96794 max_load_reduction_pct=0.422355 moved_to_replicas=385322 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] avg=131072.0000 max=132383 max_rank=1 min=128861 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96794 max_load_reduction_pct=0.422355 moved_to_replicas=385322 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[131156, 132383, 125511, 132383, 132383, 132383, 132383, 129994] avg=131072.0000 max=132383 max_rank=1 min=125511 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64168 max_load_reduction_pct=0.326470 moved_to_replicas=331102 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] avg=131072.0000 max=132383 max_rank=5 min=129093 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96958 max_load_reduction_pct=0.422768 moved_to_replicas=255371 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[131767, 98514, 131080, 98132, 131230, 162962, 229341, 65550] avg=131072.0000 max=229341 max_rank=6 min=65550 min_rank=7 imbalance_ratio=1.749733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[131767, 98514, 131080, 98132, 131230, 162962, 229341, 65550] avg=131072.0000 max=229341 max_rank=6 min=65550 min_rank=7 imbalance_ratio=1.749733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] avg=131072.0000 max=132383 max_rank=5 min=129093 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96958 max_load_reduction_pct=0.422768 moved_to_replicas=255371 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] avg=131072.0000 max=132383 max_rank=5 min=129093 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96958 max_load_reduction_pct=0.422768 moved_to_replicas=255371 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[131767, 98514, 131080, 98132, 131230, 162962, 229341, 65550] avg=131072.0000 max=229341 max_rank=6 min=65550 min_rank=7 imbalance_ratio=1.749733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[33216, 163836, 65581, 132325, 96599, 229177, 131093, 196749] avg=131072.0000 max=229177 max_rank=5 min=33216 min_rank=0 imbalance_ratio=1.748482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[131767, 129093, 131080, 130896, 131230, 132383, 132383, 129744] avg=131072.0000 max=132383 max_rank=5 min=129093 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96958 max_load_reduction_pct=0.422768 moved_to_replicas=255371 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] avg=131072.0000 max=132383 max_rank=1 min=128861 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96794 max_load_reduction_pct=0.422355 moved_to_replicas=385322 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[33216, 163836, 65581, 132325, 96599, 229177, 131093, 196749] avg=131072.0000 max=229177 max_rank=5 min=33216 min_rank=0 imbalance_ratio=1.748482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[33216, 163836, 65581, 132325, 96599, 229177, 131093, 196749] avg=131072.0000 max=229177 max_rank=5 min=33216 min_rank=0 imbalance_ratio=1.748482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] avg=131072.0000 max=132383 max_rank=1 min=128861 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96794 max_load_reduction_pct=0.422355 moved_to_replicas=385322 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[33216, 163836, 65581, 132325, 96599, 229177, 131093, 196749] avg=131072.0000 max=229177 max_rank=5 min=33216 min_rank=0 imbalance_ratio=1.748482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] avg=131072.0000 max=132383 max_rank=1 min=128861 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96794 max_load_reduction_pct=0.422355 moved_to_replicas=385322 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[128861, 132383, 129785, 132325, 129363, 132383, 131093, 132383] avg=131072.0000 max=132383 max_rank=1 min=128861 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96794 max_load_reduction_pct=0.422355 moved_to_replicas=385322 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=4205294 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=359552 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=4201670 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=361344 peak_demand=132383 required_capacity=1.010002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=4198071 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=363264 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=4205655 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=355712 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=4186673 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=374784 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=4212857 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=349696 peak_demand=132383 required_capacity=1.010002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=4189203 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=371840 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=4155009 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=404864 peak_demand=132384 required_capacity=1.010010 + [2026-08-30 05:19:11] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 746.9 | throughput per GPU (TFLOP/s/GPU): 446.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.070854E+00 | loss scale: 1.0 | grad norm: 5.396 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=143616 expected_recv=259751 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=143616 expected_recv=261983 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=143616 expected_recv=261017 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=143616 expected_recv=255335 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=143616 expected_recv=259751 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=143616 expected_recv=261983 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=143616 expected_recv=261017 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=143616 expected_recv=255335 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=257920 slack_slots=29312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=143616 expected_recv=259521 expected_overflow=0 exact_recv_per_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=143616 expected_recv=259521 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=143616 expected_recv=258399 expected_overflow=0 exact_recv_per_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=143616 expected_recv=258399 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=143616 expected_recv=255641 expected_overflow=0 exact_recv_per_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=143616 expected_recv=264532 expected_overflow=0 exact_recv_per_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=260992 slack_slots=26240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=143616 expected_recv=255641 actual_valid_recv=258432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=143616 expected_recv=264532 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=258432 slack_slots=28800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=143616 expected_recv=262182 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=143616 expected_recv=260741 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=143616 expected_recv=262978 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=143616 expected_recv=260462 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=143616 expected_recv=262182 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=143616 expected_recv=260741 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=143616 expected_recv=262978 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=143616 expected_recv=260462 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=143616 expected_recv=256491 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=263424 slack_slots=23808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=143616 expected_recv=256491 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=258688 slack_slots=28544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=143616 expected_recv=253851 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=143616 expected_recv=254706 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=143616 expected_recv=253851 actual_valid_recv=255744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=143616 expected_recv=254706 actual_valid_recv=256384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=255744 slack_slots=31488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=256384 slack_slots=30848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=143616 expected_recv=256860 expected_overflow=0 exact_recv_per_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=143616 expected_recv=256860 actual_valid_recv=258560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=258560 slack_slots=28672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=143616 expected_recv=262107 expected_overflow=0 exact_recv_per_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=143616 expected_recv=262107 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=143616 expected_recv=256998 expected_overflow=0 exact_recv_per_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=143616 expected_recv=256998 actual_valid_recv=258432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=143616 expected_recv=262489 expected_overflow=0 exact_recv_per_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=143616 expected_recv=262125 expected_overflow=0 exact_recv_per_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=258432 slack_slots=28800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=143616 expected_recv=262489 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=143616 expected_recv=262125 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=143616 expected_recv=263019 expected_overflow=0 exact_recv_per_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=143616 expected_recv=262115 expected_overflow=0 exact_recv_per_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=143616 expected_recv=261897 expected_overflow=0 exact_recv_per_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=143616 expected_recv=263019 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=143616 expected_recv=262115 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=143616 expected_recv=261897 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=143616 expected_recv=253334 expected_overflow=0 exact_recv_per_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=143616 expected_recv=253334 actual_valid_recv=254976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=143616 expected_recv=251085 expected_overflow=0 exact_recv_per_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=254976 slack_slots=32256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=143616 expected_recv=251085 actual_valid_recv=252544 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=252544 slack_slots=34688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=143616 expected_recv=262999 expected_overflow=0 exact_recv_per_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=143616 expected_recv=261991 expected_overflow=0 exact_recv_per_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=143616 expected_recv=262999 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=143616 expected_recv=262013 expected_overflow=0 exact_recv_per_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=143616 expected_recv=261991 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=143616 expected_recv=262137 expected_overflow=0 exact_recv_per_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=143616 expected_recv=262013 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=143616 expected_recv=262137 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=143616 expected_recv=260441 expected_overflow=0 exact_recv_per_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=263424 slack_slots=23808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=143616 expected_recv=260441 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=143616 expected_recv=262444 expected_overflow=0 exact_recv_per_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=143616 expected_recv=260279 expected_overflow=0 exact_recv_per_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=143616 expected_recv=262444 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=143616 expected_recv=260279 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=143616 expected_recv=260097 expected_overflow=0 exact_recv_per_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=143616 expected_recv=262763 expected_overflow=0 exact_recv_per_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=143616 expected_recv=260097 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=143616 expected_recv=262763 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=143616 expected_recv=262222 expected_overflow=0 exact_recv_per_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=143616 expected_recv=262119 expected_overflow=0 exact_recv_per_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=143616 expected_recv=262222 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=143616 expected_recv=262119 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=143616 expected_recv=256998 expected_overflow=0 exact_recv_per_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=143616 expected_recv=261867 expected_overflow=0 exact_recv_per_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=143616 expected_recv=256998 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=143616 expected_recv=261619 expected_overflow=0 exact_recv_per_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=143616 expected_recv=261867 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=259328 slack_slots=27904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=143616 expected_recv=261619 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=143616 expected_recv=259107 expected_overflow=0 exact_recv_per_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=143616 expected_recv=262335 expected_overflow=0 exact_recv_per_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=143616 expected_recv=261896 expected_overflow=0 exact_recv_per_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=143616 expected_recv=259107 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=143616 expected_recv=262335 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=143616 expected_recv=262653 expected_overflow=0 exact_recv_per_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=260864 slack_slots=26368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=143616 expected_recv=261896 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=143616 expected_recv=262653 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=143616 expected_recv=254351 expected_overflow=0 exact_recv_per_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=263424 slack_slots=23808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=143616 expected_recv=254351 actual_valid_recv=255616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=255616 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=143616 expected_recv=258054 expected_overflow=0 exact_recv_per_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=143616 expected_recv=257899 expected_overflow=0 exact_recv_per_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=143616 expected_recv=258054 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=143616 expected_recv=257899 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=143616 expected_recv=257368 expected_overflow=0 exact_recv_per_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=259712 slack_slots=27520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=260096 slack_slots=27136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=143616 expected_recv=257368 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=259072 slack_slots=28160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=143616 expected_recv=262578 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=143616 expected_recv=256227 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=143616 expected_recv=262578 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=143616 expected_recv=256227 actual_valid_recv=257536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=143616 expected_recv=254517 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=265856 slack_slots=21376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=257536 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=143616 expected_recv=254517 actual_valid_recv=256256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=256256 slack_slots=30976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=143616 expected_recv=262291 expected_overflow=0 exact_recv_per_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=143616 expected_recv=264770 expected_overflow=0 exact_recv_per_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=143616 expected_recv=262291 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=143616 expected_recv=261730 expected_overflow=0 exact_recv_per_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=143616 expected_recv=261068 expected_overflow=0 exact_recv_per_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263040 slack_slots=24192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=143616 expected_recv=264770 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=143616 expected_recv=261730 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=143616 expected_recv=261068 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=143616 expected_recv=259387 expected_overflow=0 exact_recv_per_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=265856 slack_slots=21376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=143616 expected_recv=259387 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=143616 expected_recv=258374 expected_overflow=0 exact_recv_per_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=265856 slack_slots=21376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=265856 slack_slots=21376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=143616 expected_recv=258374 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=260352 slack_slots=26880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=143616 expected_recv=258162 expected_overflow=0 exact_recv_per_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=143616 expected_recv=259814 expected_overflow=0 exact_recv_per_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=143616 expected_recv=259607 expected_overflow=0 exact_recv_per_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=143616 expected_recv=258162 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=143616 expected_recv=259814 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=143616 expected_recv=262417 expected_overflow=0 exact_recv_per_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=143616 expected_recv=259607 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=143616 expected_recv=260505 expected_overflow=0 exact_recv_per_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=259840 slack_slots=27392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=261120 slack_slots=26112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=143616 expected_recv=262417 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=143616 expected_recv=260505 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=143616 expected_recv=262344 expected_overflow=0 exact_recv_per_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=143616 expected_recv=262392 expected_overflow=0 exact_recv_per_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=143616 expected_recv=262086 expected_overflow=0 exact_recv_per_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=143616 expected_recv=262344 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=143616 expected_recv=262392 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=143616 expected_recv=263618 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=143616 expected_recv=259491 expected_overflow=0 exact_recv_per_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=143616 expected_recv=262086 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=263936 slack_slots=23296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=143616 expected_recv=258892 expected_overflow=0 exact_recv_per_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=143616 expected_recv=263618 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=143616 expected_recv=259491 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=143616 expected_recv=258892 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264832 slack_slots=22400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=260480 slack_slots=26752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=260352 slack_slots=26880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=143616 expected_recv=262316 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=143616 expected_recv=262316 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=143616 expected_recv=249993 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=143616 expected_recv=249993 actual_valid_recv=251776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=251776 slack_slots=35456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[152973, 141684, 120300, 93225, 148032, 96240, 136212, 159910] avg=131072.0000 max=159910 max_rank=7 min=93225 min_rank=3 imbalance_ratio=1.220016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=143616 expected_recv=262161 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=143616 expected_recv=262161 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[152973, 141684, 120300, 93225, 148032, 96240, 136212, 159910] avg=131072.0000 max=159910 max_rank=7 min=93225 min_rank=3 imbalance_ratio=1.220016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=263424 slack_slots=23808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=128506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27526 max_load_reduction_pct=0.172134 moved_to_replicas=150963 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[152973, 141684, 120300, 93225, 148032, 96240, 136212, 159910] avg=131072.0000 max=159910 max_rank=7 min=93225 min_rank=3 imbalance_ratio=1.220016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[152973, 141684, 120300, 93225, 148032, 96240, 136212, 159910] avg=131072.0000 max=159910 max_rank=7 min=93225 min_rank=3 imbalance_ratio=1.220016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=128506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27526 max_load_reduction_pct=0.172134 moved_to_replicas=150963 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[152973, 141684, 120300, 93225, 148032, 96240, 136212, 159910] avg=131072.0000 max=159910 max_rank=7 min=93225 min_rank=3 imbalance_ratio=1.220016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[152973, 141684, 120300, 93225, 148032, 96240, 136212, 159910] avg=131072.0000 max=159910 max_rank=7 min=93225 min_rank=3 imbalance_ratio=1.220016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[152973, 141684, 120300, 93225, 148032, 96240, 136212, 159910] avg=131072.0000 max=159910 max_rank=7 min=93225 min_rank=3 imbalance_ratio=1.220016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[55728, 132149, 148869, 194086, 158473, 100760, 144968, 113543] avg=131072.0000 max=194086 max_rank=3 min=55728 min_rank=0 imbalance_ratio=1.480759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[152973, 141684, 120300, 93225, 148032, 96240, 136212, 159910] avg=131072.0000 max=159910 max_rank=7 min=93225 min_rank=3 imbalance_ratio=1.220016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=128506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27526 max_load_reduction_pct=0.172134 moved_to_replicas=150963 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=128506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27526 max_load_reduction_pct=0.172134 moved_to_replicas=150963 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=128506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27526 max_load_reduction_pct=0.172134 moved_to_replicas=150963 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=128506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27526 max_load_reduction_pct=0.172134 moved_to_replicas=150963 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[55728, 132149, 148869, 194086, 158473, 100760, 144968, 113543] avg=131072.0000 max=194086 max_rank=3 min=55728 min_rank=0 imbalance_ratio=1.480759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] avg=131072.0000 max=132383 max_rank=2 min=126852 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61703 max_load_reduction_pct=0.317916 moved_to_replicas=232688 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=128506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27526 max_load_reduction_pct=0.172134 moved_to_replicas=150963 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132383, 129601, 128553, 132383, 128506, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=128506 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27526 max_load_reduction_pct=0.172134 moved_to_replicas=150963 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[55728, 132149, 148869, 194086, 158473, 100760, 144968, 113543] avg=131072.0000 max=194086 max_rank=3 min=55728 min_rank=0 imbalance_ratio=1.480759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[55728, 132149, 148869, 194086, 158473, 100760, 144968, 113543] avg=131072.0000 max=194086 max_rank=3 min=55728 min_rank=0 imbalance_ratio=1.480759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] avg=131072.0000 max=132383 max_rank=2 min=126852 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61703 max_load_reduction_pct=0.317916 moved_to_replicas=232688 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[55728, 132149, 148869, 194086, 158473, 100760, 144968, 113543] avg=131072.0000 max=194086 max_rank=3 min=55728 min_rank=0 imbalance_ratio=1.480759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[55728, 132149, 148869, 194086, 158473, 100760, 144968, 113543] avg=131072.0000 max=194086 max_rank=3 min=55728 min_rank=0 imbalance_ratio=1.480759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[137828, 172337, 131102, 218360, 130461, 123674, 49482, 85332] avg=131072.0000 max=218360 max_rank=3 min=49482 min_rank=6 imbalance_ratio=1.665955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[55728, 132149, 148869, 194086, 158473, 100760, 144968, 113543] avg=131072.0000 max=194086 max_rank=3 min=55728 min_rank=0 imbalance_ratio=1.480759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] avg=131072.0000 max=132383 max_rank=2 min=126852 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61703 max_load_reduction_pct=0.317916 moved_to_replicas=232688 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] avg=131072.0000 max=132383 max_rank=2 min=126852 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61703 max_load_reduction_pct=0.317916 moved_to_replicas=232688 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[55728, 132149, 148869, 194086, 158473, 100760, 144968, 113543] avg=131072.0000 max=194086 max_rank=3 min=55728 min_rank=0 imbalance_ratio=1.480759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] avg=131072.0000 max=132383 max_rank=2 min=126852 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61703 max_load_reduction_pct=0.317916 moved_to_replicas=232688 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] avg=131072.0000 max=132383 max_rank=2 min=126852 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61703 max_load_reduction_pct=0.317916 moved_to_replicas=232688 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[137828, 172337, 131102, 218360, 130461, 123674, 49482, 85332] avg=131072.0000 max=218360 max_rank=3 min=49482 min_rank=6 imbalance_ratio=1.665955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] avg=131072.0000 max=132385 max_rank=7 min=128360 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=85975 max_load_reduction_pct=0.393731 moved_to_replicas=265408 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] avg=131072.0000 max=132383 max_rank=2 min=126852 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61703 max_load_reduction_pct=0.317916 moved_to_replicas=232688 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[130017, 132149, 132383, 132383, 132381, 126852, 132382, 130029] avg=131072.0000 max=132383 max_rank=2 min=126852 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61703 max_load_reduction_pct=0.317916 moved_to_replicas=232688 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[137828, 172337, 131102, 218360, 130461, 123674, 49482, 85332] avg=131072.0000 max=218360 max_rank=3 min=49482 min_rank=6 imbalance_ratio=1.665955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[137828, 172337, 131102, 218360, 130461, 123674, 49482, 85332] avg=131072.0000 max=218360 max_rank=3 min=49482 min_rank=6 imbalance_ratio=1.665955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] avg=131072.0000 max=132385 max_rank=7 min=128360 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=85975 max_load_reduction_pct=0.393731 moved_to_replicas=265408 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[137828, 172337, 131102, 218360, 130461, 123674, 49482, 85332] avg=131072.0000 max=218360 max_rank=3 min=49482 min_rank=6 imbalance_ratio=1.665955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[137828, 172337, 131102, 218360, 130461, 123674, 49482, 85332] avg=131072.0000 max=218360 max_rank=3 min=49482 min_rank=6 imbalance_ratio=1.665955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[140920, 147182, 179119, 164540, 98216, 133695, 69797, 115107] avg=131072.0000 max=179119 max_rank=2 min=69797 min_rank=6 imbalance_ratio=1.366570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] avg=131072.0000 max=132385 max_rank=7 min=128360 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=85975 max_load_reduction_pct=0.393731 moved_to_replicas=265408 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[137828, 172337, 131102, 218360, 130461, 123674, 49482, 85332] avg=131072.0000 max=218360 max_rank=3 min=49482 min_rank=6 imbalance_ratio=1.665955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] avg=131072.0000 max=132385 max_rank=7 min=128360 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=85975 max_load_reduction_pct=0.393731 moved_to_replicas=265408 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[137828, 172337, 131102, 218360, 130461, 123674, 49482, 85332] avg=131072.0000 max=218360 max_rank=3 min=49482 min_rank=6 imbalance_ratio=1.665955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] avg=131072.0000 max=132385 max_rank=7 min=128360 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=85975 max_load_reduction_pct=0.393731 moved_to_replicas=265408 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[140920, 147182, 179119, 164540, 98216, 133695, 69797, 115107] avg=131072.0000 max=179119 max_rank=2 min=69797 min_rank=6 imbalance_ratio=1.366570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] avg=131072.0000 max=132384 max_rank=0 min=124955 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=46735 max_load_reduction_pct=0.260916 moved_to_replicas=213941 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] avg=131072.0000 max=132385 max_rank=7 min=128360 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=85975 max_load_reduction_pct=0.393731 moved_to_replicas=265408 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] avg=131072.0000 max=132385 max_rank=7 min=128360 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=85975 max_load_reduction_pct=0.393731 moved_to_replicas=265408 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 131102, 132383, 130461, 129120, 128360, 132385] avg=131072.0000 max=132385 max_rank=7 min=128360 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=85975 max_load_reduction_pct=0.393731 moved_to_replicas=265408 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] avg=131072.0000 max=132384 max_rank=0 min=124955 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=46735 max_load_reduction_pct=0.260916 moved_to_replicas=213941 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[140920, 147182, 179119, 164540, 98216, 133695, 69797, 115107] avg=131072.0000 max=179119 max_rank=2 min=69797 min_rank=6 imbalance_ratio=1.366570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[140920, 147182, 179119, 164540, 98216, 133695, 69797, 115107] avg=131072.0000 max=179119 max_rank=2 min=69797 min_rank=6 imbalance_ratio=1.366570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[140920, 147182, 179119, 164540, 98216, 133695, 69797, 115107] avg=131072.0000 max=179119 max_rank=2 min=69797 min_rank=6 imbalance_ratio=1.366570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[104668, 98269, 133716, 98330, 131050, 131058, 161761, 189724] avg=131072.0000 max=189724 max_rank=7 min=98269 min_rank=1 imbalance_ratio=1.447479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[140920, 147182, 179119, 164540, 98216, 133695, 69797, 115107] avg=131072.0000 max=179119 max_rank=2 min=69797 min_rank=6 imbalance_ratio=1.366570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[140920, 147182, 179119, 164540, 98216, 133695, 69797, 115107] avg=131072.0000 max=179119 max_rank=2 min=69797 min_rank=6 imbalance_ratio=1.366570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] avg=131072.0000 max=132384 max_rank=0 min=124955 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=46735 max_load_reduction_pct=0.260916 moved_to_replicas=213941 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] avg=131072.0000 max=132384 max_rank=0 min=124955 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=46735 max_load_reduction_pct=0.260916 moved_to_replicas=213941 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[140920, 147182, 179119, 164540, 98216, 133695, 69797, 115107] avg=131072.0000 max=179119 max_rank=2 min=69797 min_rank=6 imbalance_ratio=1.366570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[104668, 98269, 133716, 98330, 131050, 131058, 161761, 189724] avg=131072.0000 max=189724 max_rank=7 min=98269 min_rank=1 imbalance_ratio=1.447479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] avg=131072.0000 max=132384 max_rank=0 min=124955 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=46735 max_load_reduction_pct=0.260916 moved_to_replicas=213941 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125905 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57340 max_load_reduction_pct=0.302229 moved_to_replicas=171640 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] avg=131072.0000 max=132384 max_rank=0 min=124955 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=46735 max_load_reduction_pct=0.260916 moved_to_replicas=213941 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] avg=131072.0000 max=132384 max_rank=0 min=124955 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=46735 max_load_reduction_pct=0.260916 moved_to_replicas=213941 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[132384, 132382, 132383, 132383, 129323, 132383, 132383, 124955] avg=131072.0000 max=132384 max_rank=0 min=124955 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=46735 max_load_reduction_pct=0.260916 moved_to_replicas=213941 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125905 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57340 max_load_reduction_pct=0.302229 moved_to_replicas=171640 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[104668, 98269, 133716, 98330, 131050, 131058, 161761, 189724] avg=131072.0000 max=189724 max_rank=7 min=98269 min_rank=1 imbalance_ratio=1.447479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[104668, 98269, 133716, 98330, 131050, 131058, 161761, 189724] avg=131072.0000 max=189724 max_rank=7 min=98269 min_rank=1 imbalance_ratio=1.447479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[104668, 98269, 133716, 98330, 131050, 131058, 161761, 189724] avg=131072.0000 max=189724 max_rank=7 min=98269 min_rank=1 imbalance_ratio=1.447479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[147246, 78162, 145301, 131243, 131072, 246833, 103264, 65455] avg=131072.0000 max=246833 max_rank=5 min=65455 min_rank=7 imbalance_ratio=1.883186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[104668, 98269, 133716, 98330, 131050, 131058, 161761, 189724] avg=131072.0000 max=189724 max_rank=7 min=98269 min_rank=1 imbalance_ratio=1.447479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[104668, 98269, 133716, 98330, 131050, 131058, 161761, 189724] avg=131072.0000 max=189724 max_rank=7 min=98269 min_rank=1 imbalance_ratio=1.447479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125905 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57340 max_load_reduction_pct=0.302229 moved_to_replicas=171640 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125905 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57340 max_load_reduction_pct=0.302229 moved_to_replicas=171640 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[104668, 98269, 133716, 98330, 131050, 131058, 161761, 189724] avg=131072.0000 max=189724 max_rank=7 min=98269 min_rank=1 imbalance_ratio=1.447479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[147246, 78162, 145301, 131243, 131072, 246833, 103264, 65455] avg=131072.0000 max=246833 max_rank=5 min=65455 min_rank=7 imbalance_ratio=1.883186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125905 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57340 max_load_reduction_pct=0.302229 moved_to_replicas=171640 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125905 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57340 max_load_reduction_pct=0.302229 moved_to_replicas=171640 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] avg=131072.0000 max=132384 max_rank=5 min=125787 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=114449 max_load_reduction_pct=0.463670 moved_to_replicas=281966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125905 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57340 max_load_reduction_pct=0.302229 moved_to_replicas=171640 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[147246, 78162, 145301, 131243, 131072, 246833, 103264, 65455] avg=131072.0000 max=246833 max_rank=5 min=65455 min_rank=7 imbalance_ratio=1.883186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[147246, 78162, 145301, 131243, 131072, 246833, 103264, 65455] avg=131072.0000 max=246833 max_rank=5 min=65455 min_rank=7 imbalance_ratio=1.883186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[132383, 131030, 132383, 125905, 131050, 131058, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=125905 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57340 max_load_reduction_pct=0.302229 moved_to_replicas=171640 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] avg=131072.0000 max=132384 max_rank=5 min=125787 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=114449 max_load_reduction_pct=0.463670 moved_to_replicas=281966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[147246, 78162, 145301, 131243, 131072, 246833, 103264, 65455] avg=131072.0000 max=246833 max_rank=5 min=65455 min_rank=7 imbalance_ratio=1.883186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[65068, 98382, 163694, 165627, 99926, 226112, 164270, 65497] avg=131072.0000 max=226112 max_rank=5 min=65068 min_rank=0 imbalance_ratio=1.725098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[147246, 78162, 145301, 131243, 131072, 246833, 103264, 65455] avg=131072.0000 max=246833 max_rank=5 min=65455 min_rank=7 imbalance_ratio=1.883186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[147246, 78162, 145301, 131243, 131072, 246833, 103264, 65455] avg=131072.0000 max=246833 max_rank=5 min=65455 min_rank=7 imbalance_ratio=1.883186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] avg=131072.0000 max=132384 max_rank=5 min=125787 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=114449 max_load_reduction_pct=0.463670 moved_to_replicas=281966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] avg=131072.0000 max=132384 max_rank=5 min=125787 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=114449 max_load_reduction_pct=0.463670 moved_to_replicas=281966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[65068, 98382, 163694, 165627, 99926, 226112, 164270, 65497] avg=131072.0000 max=226112 max_rank=5 min=65068 min_rank=0 imbalance_ratio=1.725098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[147246, 78162, 145301, 131243, 131072, 246833, 103264, 65455] avg=131072.0000 max=246833 max_rank=5 min=65455 min_rank=7 imbalance_ratio=1.883186 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] avg=131072.0000 max=132384 max_rank=5 min=125787 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=114449 max_load_reduction_pct=0.463670 moved_to_replicas=281966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] avg=131072.0000 max=132384 max_rank=5 min=126036 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93728 max_load_reduction_pct=0.414520 moved_to_replicas=379937 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] avg=131072.0000 max=132384 max_rank=5 min=125787 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=114449 max_load_reduction_pct=0.463670 moved_to_replicas=281966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] avg=131072.0000 max=132384 max_rank=5 min=125787 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=114449 max_load_reduction_pct=0.463670 moved_to_replicas=281966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[65068, 98382, 163694, 165627, 99926, 226112, 164270, 65497] avg=131072.0000 max=226112 max_rank=5 min=65068 min_rank=0 imbalance_ratio=1.725098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[65068, 98382, 163694, 165627, 99926, 226112, 164270, 65497] avg=131072.0000 max=226112 max_rank=5 min=65068 min_rank=0 imbalance_ratio=1.725098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] avg=131072.0000 max=132384 max_rank=5 min=126036 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93728 max_load_reduction_pct=0.414520 moved_to_replicas=379937 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[132382, 125787, 132382, 131243, 131072, 132384, 132348, 130978] avg=131072.0000 max=132384 max_rank=5 min=125787 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=114449 max_load_reduction_pct=0.463670 moved_to_replicas=281966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[65068, 98382, 163694, 165627, 99926, 226112, 164270, 65497] avg=131072.0000 max=226112 max_rank=5 min=65068 min_rank=0 imbalance_ratio=1.725098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[115060, 81622, 196149, 98308, 98322, 131108, 65760, 262247] avg=131072.0000 max=262247 max_rank=7 min=65760 min_rank=6 imbalance_ratio=2.000786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[65068, 98382, 163694, 165627, 99926, 226112, 164270, 65497] avg=131072.0000 max=226112 max_rank=5 min=65068 min_rank=0 imbalance_ratio=1.725098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[65068, 98382, 163694, 165627, 99926, 226112, 164270, 65497] avg=131072.0000 max=226112 max_rank=5 min=65068 min_rank=0 imbalance_ratio=1.725098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] avg=131072.0000 max=132384 max_rank=5 min=126036 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93728 max_load_reduction_pct=0.414520 moved_to_replicas=379937 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] avg=131072.0000 max=132384 max_rank=5 min=126036 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93728 max_load_reduction_pct=0.414520 moved_to_replicas=379937 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] avg=131072.0000 max=132384 max_rank=1 min=128059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=129863 max_load_reduction_pct=0.495193 moved_to_replicas=389108 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] avg=131072.0000 max=132384 max_rank=5 min=126036 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93728 max_load_reduction_pct=0.414520 moved_to_replicas=379937 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[115060, 81622, 196149, 98308, 98322, 131108, 65760, 262247] avg=131072.0000 max=262247 max_rank=7 min=65760 min_rank=6 imbalance_ratio=2.000786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[65068, 98382, 163694, 165627, 99926, 226112, 164270, 65497] avg=131072.0000 max=226112 max_rank=5 min=65068 min_rank=0 imbalance_ratio=1.725098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] avg=131072.0000 max=132384 max_rank=5 min=126036 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93728 max_load_reduction_pct=0.414520 moved_to_replicas=379937 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] avg=131072.0000 max=132384 max_rank=5 min=126036 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93728 max_load_reduction_pct=0.414520 moved_to_replicas=379937 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[115060, 81622, 196149, 98308, 98322, 131108, 65760, 262247] avg=131072.0000 max=262247 max_rank=7 min=65760 min_rank=6 imbalance_ratio=2.000786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[115060, 81622, 196149, 98308, 98322, 131108, 65760, 262247] avg=131072.0000 max=262247 max_rank=7 min=65760 min_rank=6 imbalance_ratio=2.000786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] avg=131072.0000 max=132384 max_rank=1 min=128059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=129863 max_load_reduction_pct=0.495193 moved_to_replicas=389108 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[126036, 130753, 132383, 132383, 131237, 132384, 132383, 131017] avg=131072.0000 max=132384 max_rank=5 min=126036 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=93728 max_load_reduction_pct=0.414520 moved_to_replicas=379937 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[98242, 130954, 89608, 131527, 112402, 98083, 191274, 196486] avg=131072.0000 max=196486 max_rank=7 min=89608 min_rank=2 imbalance_ratio=1.499069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[115060, 81622, 196149, 98308, 98322, 131108, 65760, 262247] avg=131072.0000 max=262247 max_rank=7 min=65760 min_rank=6 imbalance_ratio=2.000786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] avg=131072.0000 max=132384 max_rank=1 min=128059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=129863 max_load_reduction_pct=0.495193 moved_to_replicas=389108 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] avg=131072.0000 max=132384 max_rank=1 min=128059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=129863 max_load_reduction_pct=0.495193 moved_to_replicas=389108 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[115060, 81622, 196149, 98308, 98322, 131108, 65760, 262247] avg=131072.0000 max=262247 max_rank=7 min=65760 min_rank=6 imbalance_ratio=2.000786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[115060, 81622, 196149, 98308, 98322, 131108, 65760, 262247] avg=131072.0000 max=262247 max_rank=7 min=65760 min_rank=6 imbalance_ratio=2.000786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] avg=131072.0000 max=132384 max_rank=4 min=128519 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64102 max_load_reduction_pct=0.326242 moved_to_replicas=247549 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] avg=131072.0000 max=132384 max_rank=1 min=128059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=129863 max_load_reduction_pct=0.495193 moved_to_replicas=389108 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[98242, 130954, 89608, 131527, 112402, 98083, 191274, 196486] avg=131072.0000 max=196486 max_rank=7 min=89608 min_rank=2 imbalance_ratio=1.499069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[115060, 81622, 196149, 98308, 98322, 131108, 65760, 262247] avg=131072.0000 max=262247 max_rank=7 min=65760 min_rank=6 imbalance_ratio=2.000786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] avg=131072.0000 max=132384 max_rank=1 min=128059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=129863 max_load_reduction_pct=0.495193 moved_to_replicas=389108 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] avg=131072.0000 max=132384 max_rank=1 min=128059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=129863 max_load_reduction_pct=0.495193 moved_to_replicas=389108 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[98242, 130954, 89608, 131527, 112402, 98083, 191274, 196486] avg=131072.0000 max=196486 max_rank=7 min=89608 min_rank=2 imbalance_ratio=1.499069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[98242, 130954, 89608, 131527, 112402, 98083, 191274, 196486] avg=131072.0000 max=196486 max_rank=7 min=89608 min_rank=2 imbalance_ratio=1.499069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] avg=131072.0000 max=132384 max_rank=4 min=128519 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64102 max_load_reduction_pct=0.326242 moved_to_replicas=247549 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[128059, 132384, 132383, 131064, 129912, 131108, 131283, 132383] avg=131072.0000 max=132384 max_rank=1 min=128059 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=129863 max_load_reduction_pct=0.495193 moved_to_replicas=389108 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[98242, 130954, 89608, 131527, 112402, 98083, 191274, 196486] avg=131072.0000 max=196486 max_rank=7 min=89608 min_rank=2 imbalance_ratio=1.499069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 161797, 98279, 36773, 163727, 130740, 199912] avg=131072.0000 max=199912 max_rank=7 min=36773 min_rank=4 imbalance_ratio=1.525208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] avg=131072.0000 max=132384 max_rank=4 min=128519 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64102 max_load_reduction_pct=0.326242 moved_to_replicas=247549 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] avg=131072.0000 max=132384 max_rank=4 min=128519 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64102 max_load_reduction_pct=0.326242 moved_to_replicas=247549 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[98242, 130954, 89608, 131527, 112402, 98083, 191274, 196486] avg=131072.0000 max=196486 max_rank=7 min=89608 min_rank=2 imbalance_ratio=1.499069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[98242, 130954, 89608, 131527, 112402, 98083, 191274, 196486] avg=131072.0000 max=196486 max_rank=7 min=89608 min_rank=2 imbalance_ratio=1.499069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 161797, 98279, 36773, 163727, 130740, 199912] avg=131072.0000 max=199912 max_rank=7 min=36773 min_rank=4 imbalance_ratio=1.525208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] avg=131072.0000 max=132384 max_rank=4 min=128519 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64102 max_load_reduction_pct=0.326242 moved_to_replicas=247549 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] avg=131072.0000 max=132383 max_rank=2 min=126222 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=67529 max_load_reduction_pct=0.337794 moved_to_replicas=252691 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[98242, 130954, 89608, 131527, 112402, 98083, 191274, 196486] avg=131072.0000 max=196486 max_rank=7 min=89608 min_rank=2 imbalance_ratio=1.499069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] avg=131072.0000 max=132384 max_rank=4 min=128519 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64102 max_load_reduction_pct=0.326242 moved_to_replicas=247549 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] avg=131072.0000 max=132384 max_rank=4 min=128519 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64102 max_load_reduction_pct=0.326242 moved_to_replicas=247549 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 161797, 98279, 36773, 163727, 130740, 199912] avg=131072.0000 max=199912 max_rank=7 min=36773 min_rank=4 imbalance_ratio=1.525208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 161797, 98279, 36773, 163727, 130740, 199912] avg=131072.0000 max=199912 max_rank=7 min=36773 min_rank=4 imbalance_ratio=1.525208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] avg=131072.0000 max=132383 max_rank=2 min=126222 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=67529 max_load_reduction_pct=0.337794 moved_to_replicas=252691 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 161797, 98279, 36773, 163727, 130740, 199912] avg=131072.0000 max=199912 max_rank=7 min=36773 min_rank=4 imbalance_ratio=1.525208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[131003, 130954, 128519, 131527, 132384, 129425, 132381, 132383] avg=131072.0000 max=132384 max_rank=4 min=128519 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64102 max_load_reduction_pct=0.326242 moved_to_replicas=247549 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[65137, 228769, 44364, 195832, 70619, 222875, 56763, 164217] avg=131072.0000 max=228769 max_rank=1 min=44364 min_rank=2 imbalance_ratio=1.745369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] avg=131072.0000 max=132383 max_rank=2 min=126222 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=67529 max_load_reduction_pct=0.337794 moved_to_replicas=252691 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] avg=131072.0000 max=132383 max_rank=2 min=126222 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=67529 max_load_reduction_pct=0.337794 moved_to_replicas=252691 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 161797, 98279, 36773, 163727, 130740, 199912] avg=131072.0000 max=199912 max_rank=7 min=36773 min_rank=4 imbalance_ratio=1.525208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 161797, 98279, 36773, 163727, 130740, 199912] avg=131072.0000 max=199912 max_rank=7 min=36773 min_rank=4 imbalance_ratio=1.525208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[65137, 228769, 44364, 195832, 70619, 222875, 56763, 164217] avg=131072.0000 max=228769 max_rank=1 min=44364 min_rank=2 imbalance_ratio=1.745369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] avg=131072.0000 max=132383 max_rank=2 min=126222 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=67529 max_load_reduction_pct=0.337794 moved_to_replicas=252691 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] avg=131072.0000 max=132384 max_rank=2 min=125515 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96385 max_load_reduction_pct=0.421320 moved_to_replicas=565614 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] avg=131072.0000 max=132383 max_rank=2 min=126222 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=67529 max_load_reduction_pct=0.337794 moved_to_replicas=252691 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[65137, 228769, 44364, 195832, 70619, 222875, 56763, 164217] avg=131072.0000 max=228769 max_rank=1 min=44364 min_rank=2 imbalance_ratio=1.745369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 161797, 98279, 36773, 163727, 130740, 199912] avg=131072.0000 max=199912 max_rank=7 min=36773 min_rank=4 imbalance_ratio=1.525208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] avg=131072.0000 max=132383 max_rank=2 min=126222 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=67529 max_load_reduction_pct=0.337794 moved_to_replicas=252691 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[65137, 228769, 44364, 195832, 70619, 222875, 56763, 164217] avg=131072.0000 max=228769 max_rank=1 min=44364 min_rank=2 imbalance_ratio=1.745369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] avg=131072.0000 max=132384 max_rank=2 min=125515 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96385 max_load_reduction_pct=0.421320 moved_to_replicas=565614 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[65137, 228769, 44364, 195832, 70619, 222875, 56763, 164217] avg=131072.0000 max=228769 max_rank=1 min=44364 min_rank=2 imbalance_ratio=1.745369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[164431, 167598, 62677, 163629, 124241, 163873, 131682, 70445] avg=131072.0000 max=167598 max_rank=1 min=62677 min_rank=2 imbalance_ratio=1.278671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] avg=131072.0000 max=132384 max_rank=2 min=125515 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96385 max_load_reduction_pct=0.421320 moved_to_replicas=565614 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[131126, 126222, 132383, 131638, 131701, 132383, 130740, 132383] avg=131072.0000 max=132383 max_rank=2 min=126222 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=67529 max_load_reduction_pct=0.337794 moved_to_replicas=252691 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] avg=131072.0000 max=132384 max_rank=2 min=125515 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96385 max_load_reduction_pct=0.421320 moved_to_replicas=565614 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[65137, 228769, 44364, 195832, 70619, 222875, 56763, 164217] avg=131072.0000 max=228769 max_rank=1 min=44364 min_rank=2 imbalance_ratio=1.745369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] avg=131072.0000 max=132384 max_rank=2 min=125515 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96385 max_load_reduction_pct=0.421320 moved_to_replicas=565614 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] avg=131072.0000 max=132383 max_rank=0 min=126921 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=35215 max_load_reduction_pct=0.210116 moved_to_replicas=259277 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[164431, 167598, 62677, 163629, 124241, 163873, 131682, 70445] avg=131072.0000 max=167598 max_rank=1 min=62677 min_rank=2 imbalance_ratio=1.278671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[65137, 228769, 44364, 195832, 70619, 222875, 56763, 164217] avg=131072.0000 max=228769 max_rank=1 min=44364 min_rank=2 imbalance_ratio=1.745369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] avg=131072.0000 max=132384 max_rank=2 min=125515 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96385 max_load_reduction_pct=0.421320 moved_to_replicas=565614 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[164431, 167598, 62677, 163629, 124241, 163873, 131682, 70445] avg=131072.0000 max=167598 max_rank=1 min=62677 min_rank=2 imbalance_ratio=1.278671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[164431, 167598, 62677, 163629, 124241, 163873, 131682, 70445] avg=131072.0000 max=167598 max_rank=1 min=62677 min_rank=2 imbalance_ratio=1.278671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[65137, 228769, 44364, 195832, 70619, 222875, 56763, 164217] avg=131072.0000 max=228769 max_rank=1 min=44364 min_rank=2 imbalance_ratio=1.745369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] avg=131072.0000 max=132383 max_rank=0 min=126921 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=35215 max_load_reduction_pct=0.210116 moved_to_replicas=259277 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] avg=131072.0000 max=132384 max_rank=2 min=125515 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96385 max_load_reduction_pct=0.421320 moved_to_replicas=565614 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[164431, 167598, 62677, 163629, 124241, 163873, 131682, 70445] avg=131072.0000 max=167598 max_rank=1 min=62677 min_rank=2 imbalance_ratio=1.278671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[140892, 196519, 65640, 163644, 130653, 89980, 131208, 130040] avg=131072.0000 max=196519 max_rank=1 min=65640 min_rank=2 imbalance_ratio=1.499321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] avg=131072.0000 max=132383 max_rank=0 min=126921 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=35215 max_load_reduction_pct=0.210116 moved_to_replicas=259277 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] avg=131072.0000 max=132383 max_rank=0 min=126921 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=35215 max_load_reduction_pct=0.210116 moved_to_replicas=259277 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[128762, 132383, 132384, 132383, 132383, 132383, 125515, 132383] avg=131072.0000 max=132384 max_rank=2 min=125515 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96385 max_load_reduction_pct=0.421320 moved_to_replicas=565614 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] avg=131072.0000 max=132383 max_rank=0 min=126921 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=35215 max_load_reduction_pct=0.210116 moved_to_replicas=259277 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[164431, 167598, 62677, 163629, 124241, 163873, 131682, 70445] avg=131072.0000 max=167598 max_rank=1 min=62677 min_rank=2 imbalance_ratio=1.278671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] avg=131072.0000 max=132386 max_rank=0 min=129662 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64133 max_load_reduction_pct=0.326345 moved_to_replicas=209558 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[140892, 196519, 65640, 163644, 130653, 89980, 131208, 130040] avg=131072.0000 max=196519 max_rank=1 min=65640 min_rank=2 imbalance_ratio=1.499321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[164431, 167598, 62677, 163629, 124241, 163873, 131682, 70445] avg=131072.0000 max=167598 max_rank=1 min=62677 min_rank=2 imbalance_ratio=1.278671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] avg=131072.0000 max=132383 max_rank=0 min=126921 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=35215 max_load_reduction_pct=0.210116 moved_to_replicas=259277 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] avg=131072.0000 max=132386 max_rank=0 min=129662 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64133 max_load_reduction_pct=0.326345 moved_to_replicas=209558 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[140892, 196519, 65640, 163644, 130653, 89980, 131208, 130040] avg=131072.0000 max=196519 max_rank=1 min=65640 min_rank=2 imbalance_ratio=1.499321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[140892, 196519, 65640, 163644, 130653, 89980, 131208, 130040] avg=131072.0000 max=196519 max_rank=1 min=65640 min_rank=2 imbalance_ratio=1.499321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] avg=131072.0000 max=132383 max_rank=0 min=126921 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=35215 max_load_reduction_pct=0.210116 moved_to_replicas=259277 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[164431, 167598, 62677, 163629, 124241, 163873, 131682, 70445] avg=131072.0000 max=167598 max_rank=1 min=62677 min_rank=2 imbalance_ratio=1.278671 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[140892, 196519, 65640, 163644, 130653, 89980, 131208, 130040] avg=131072.0000 max=196519 max_rank=1 min=65640 min_rank=2 imbalance_ratio=1.499321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[65659, 163794, 97997, 163764, 98259, 196856, 195439, 66808] avg=131072.0000 max=196856 max_rank=5 min=65659 min_rank=0 imbalance_ratio=1.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] avg=131072.0000 max=132386 max_rank=0 min=129662 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64133 max_load_reduction_pct=0.326345 moved_to_replicas=209558 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] avg=131072.0000 max=132386 max_rank=0 min=129662 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64133 max_load_reduction_pct=0.326345 moved_to_replicas=209558 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[140892, 196519, 65640, 163644, 130653, 89980, 131208, 130040] avg=131072.0000 max=196519 max_rank=1 min=65640 min_rank=2 imbalance_ratio=1.499321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[65659, 163794, 97997, 163764, 98259, 196856, 195439, 66808] avg=131072.0000 max=196856 max_rank=5 min=65659 min_rank=0 imbalance_ratio=1.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] avg=131072.0000 max=132386 max_rank=0 min=129662 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64133 max_load_reduction_pct=0.326345 moved_to_replicas=209558 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132383, 126921, 132383, 128058, 132383, 131682, 132383] avg=131072.0000 max=132383 max_rank=0 min=126921 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=35215 max_load_reduction_pct=0.210116 moved_to_replicas=259277 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] avg=131072.0000 max=132383 max_rank=1 min=128554 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64473 max_load_reduction_pct=0.327514 moved_to_replicas=381662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[140892, 196519, 65640, 163644, 130653, 89980, 131208, 130040] avg=131072.0000 max=196519 max_rank=1 min=65640 min_rank=2 imbalance_ratio=1.499321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] avg=131072.0000 max=132386 max_rank=0 min=129662 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64133 max_load_reduction_pct=0.326345 moved_to_replicas=209558 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[65659, 163794, 97997, 163764, 98259, 196856, 195439, 66808] avg=131072.0000 max=196856 max_rank=5 min=65659 min_rank=0 imbalance_ratio=1.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] avg=131072.0000 max=132383 max_rank=1 min=128554 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64473 max_load_reduction_pct=0.327514 moved_to_replicas=381662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[65659, 163794, 97997, 163764, 98259, 196856, 195439, 66808] avg=131072.0000 max=196856 max_rank=5 min=65659 min_rank=0 imbalance_ratio=1.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] avg=131072.0000 max=132386 max_rank=0 min=129662 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64133 max_load_reduction_pct=0.326345 moved_to_replicas=209558 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[65659, 163794, 97997, 163764, 98259, 196856, 195439, 66808] avg=131072.0000 max=196856 max_rank=5 min=65659 min_rank=0 imbalance_ratio=1.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[131068, 98431, 129584, 69993, 131159, 193422, 229186, 65733] avg=131072.0000 max=229186 max_rank=6 min=65733 min_rank=7 imbalance_ratio=1.748550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[140892, 196519, 65640, 163644, 130653, 89980, 131208, 130040] avg=131072.0000 max=196519 max_rank=1 min=65640 min_rank=2 imbalance_ratio=1.499321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] avg=131072.0000 max=132383 max_rank=1 min=128554 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64473 max_load_reduction_pct=0.327514 moved_to_replicas=381662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] avg=131072.0000 max=132383 max_rank=1 min=128554 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64473 max_load_reduction_pct=0.327514 moved_to_replicas=381662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] avg=131072.0000 max=132383 max_rank=1 min=128554 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64473 max_load_reduction_pct=0.327514 moved_to_replicas=381662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] avg=131072.0000 max=132383 max_rank=6 min=129584 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96803 max_load_reduction_pct=0.422377 moved_to_replicas=318534 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[65659, 163794, 97997, 163764, 98259, 196856, 195439, 66808] avg=131072.0000 max=196856 max_rank=5 min=65659 min_rank=0 imbalance_ratio=1.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[131068, 98431, 129584, 69993, 131159, 193422, 229186, 65733] avg=131072.0000 max=229186 max_rank=6 min=65733 min_rank=7 imbalance_ratio=1.748550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[132386, 132383, 129662, 132383, 130653, 129861, 131208, 130040] avg=131072.0000 max=132386 max_rank=0 min=129662 min_rank=2 imbalance_ratio=1.010025 max_load_reduction=64133 max_load_reduction_pct=0.326345 moved_to_replicas=209558 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[65659, 163794, 97997, 163764, 98259, 196856, 195439, 66808] avg=131072.0000 max=196856 max_rank=5 min=65659 min_rank=0 imbalance_ratio=1.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[131068, 98431, 129584, 69993, 131159, 193422, 229186, 65733] avg=131072.0000 max=229186 max_rank=6 min=65733 min_rank=7 imbalance_ratio=1.748550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[131068, 98431, 129584, 69993, 131159, 193422, 229186, 65733] avg=131072.0000 max=229186 max_rank=6 min=65733 min_rank=7 imbalance_ratio=1.748550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] avg=131072.0000 max=132383 max_rank=6 min=129584 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96803 max_load_reduction_pct=0.422377 moved_to_replicas=318534 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] avg=131072.0000 max=132383 max_rank=1 min=128554 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64473 max_load_reduction_pct=0.327514 moved_to_replicas=381662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[131068, 98431, 129584, 69993, 131159, 193422, 229186, 65733] avg=131072.0000 max=229186 max_rank=6 min=65733 min_rank=7 imbalance_ratio=1.748550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[68566, 196588, 65664, 132935, 57593, 198542, 131245, 197443] avg=131072.0000 max=198542 max_rank=5 min=57593 min_rank=4 imbalance_ratio=1.514755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] avg=131072.0000 max=132383 max_rank=1 min=128554 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64473 max_load_reduction_pct=0.327514 moved_to_replicas=381662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[65659, 163794, 97997, 163764, 98259, 196856, 195439, 66808] avg=131072.0000 max=196856 max_rank=5 min=65659 min_rank=0 imbalance_ratio=1.501892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] avg=131072.0000 max=132383 max_rank=6 min=129584 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96803 max_load_reduction_pct=0.422377 moved_to_replicas=318534 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] avg=131072.0000 max=132383 max_rank=6 min=129584 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96803 max_load_reduction_pct=0.422377 moved_to_replicas=318534 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] avg=131072.0000 max=132383 max_rank=6 min=129584 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96803 max_load_reduction_pct=0.422377 moved_to_replicas=318534 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] avg=131072.0000 max=132383 max_rank=0 min=124254 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66159 max_load_reduction_pct=0.333224 moved_to_replicas=389839 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[68566, 196588, 65664, 132935, 57593, 198542, 131245, 197443] avg=131072.0000 max=198542 max_rank=5 min=57593 min_rank=4 imbalance_ratio=1.514755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[131068, 98431, 129584, 69993, 131159, 193422, 229186, 65733] avg=131072.0000 max=229186 max_rank=6 min=65733 min_rank=7 imbalance_ratio=1.748550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[129830, 132383, 130758, 132383, 128554, 132383, 132383, 129902] avg=131072.0000 max=132383 max_rank=1 min=128554 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64473 max_load_reduction_pct=0.327514 moved_to_replicas=381662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[131068, 98431, 129584, 69993, 131159, 193422, 229186, 65733] avg=131072.0000 max=229186 max_rank=6 min=65733 min_rank=7 imbalance_ratio=1.748550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[68566, 196588, 65664, 132935, 57593, 198542, 131245, 197443] avg=131072.0000 max=198542 max_rank=5 min=57593 min_rank=4 imbalance_ratio=1.514755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[68566, 196588, 65664, 132935, 57593, 198542, 131245, 197443] avg=131072.0000 max=198542 max_rank=5 min=57593 min_rank=4 imbalance_ratio=1.514755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] avg=131072.0000 max=132383 max_rank=0 min=124254 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66159 max_load_reduction_pct=0.333224 moved_to_replicas=389839 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[68566, 196588, 65664, 132935, 57593, 198542, 131245, 197443] avg=131072.0000 max=198542 max_rank=5 min=57593 min_rank=4 imbalance_ratio=1.514755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] avg=131072.0000 max=132383 max_rank=6 min=129584 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96803 max_load_reduction_pct=0.422377 moved_to_replicas=318534 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] avg=131072.0000 max=132383 max_rank=6 min=129584 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96803 max_load_reduction_pct=0.422377 moved_to_replicas=318534 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[131068, 98431, 129584, 69993, 131159, 193422, 229186, 65733] avg=131072.0000 max=229186 max_rank=6 min=65733 min_rank=7 imbalance_ratio=1.748550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] avg=131072.0000 max=132383 max_rank=0 min=124254 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66159 max_load_reduction_pct=0.333224 moved_to_replicas=389839 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] avg=131072.0000 max=132383 max_rank=0 min=124254 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66159 max_load_reduction_pct=0.333224 moved_to_replicas=389839 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] avg=131072.0000 max=132383 max_rank=0 min=124254 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66159 max_load_reduction_pct=0.333224 moved_to_replicas=389839 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[68566, 196588, 65664, 132935, 57593, 198542, 131245, 197443] avg=131072.0000 max=198542 max_rank=5 min=57593 min_rank=4 imbalance_ratio=1.514755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[131068, 129715, 129584, 131033, 131159, 132381, 132383, 131253] avg=131072.0000 max=132383 max_rank=6 min=129584 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96803 max_load_reduction_pct=0.422377 moved_to_replicas=318534 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[68566, 196588, 65664, 132935, 57593, 198542, 131245, 197443] avg=131072.0000 max=198542 max_rank=5 min=57593 min_rank=4 imbalance_ratio=1.514755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] avg=131072.0000 max=132383 max_rank=0 min=124254 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66159 max_load_reduction_pct=0.333224 moved_to_replicas=389839 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] avg=131072.0000 max=132383 max_rank=0 min=124254 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66159 max_load_reduction_pct=0.333224 moved_to_replicas=389839 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[68566, 196588, 65664, 132935, 57593, 198542, 131245, 197443] avg=131072.0000 max=198542 max_rank=5 min=57593 min_rank=4 imbalance_ratio=1.514755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[132383, 132383, 131162, 132383, 124254, 132383, 131245, 132383] avg=131072.0000 max=132383 max_rank=0 min=124254 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66159 max_load_reduction_pct=0.333224 moved_to_replicas=389839 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=4209783 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=361344 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=4207558 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=362752 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=4201372 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368896 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=4190812 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=382336 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=4171985 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=396928 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=4193381 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376960 peak_demand=132386 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=4187352 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=379008 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=4192189 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=375552 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:19:12] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 564.1 | throughput per GPU (TFLOP/s/GPU): 591.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.918007E+00 | loss scale: 1.0 | grad norm: 5.199 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=143616 expected_recv=262009 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=143616 expected_recv=254605 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=143616 expected_recv=262009 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=143616 expected_recv=263007 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=143616 expected_recv=258467 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=265216 slack_slots=22016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=143616 expected_recv=254605 actual_valid_recv=256896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=143616 expected_recv=263007 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=256896 slack_slots=30336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=264832 slack_slots=22400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=143616 expected_recv=258467 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=143616 expected_recv=263821 expected_overflow=0 exact_recv_per_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=143616 expected_recv=255391 expected_overflow=0 exact_recv_per_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=143616 expected_recv=263821 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=143616 expected_recv=255391 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=143616 expected_recv=255617 expected_overflow=0 exact_recv_per_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=143616 expected_recv=263258 expected_overflow=0 exact_recv_per_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=258304 slack_slots=28928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=143616 expected_recv=255617 actual_valid_recv=258048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=143616 expected_recv=263258 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=258048 slack_slots=29184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267904 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=143616 expected_recv=255497 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=143616 expected_recv=262165 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=143616 expected_recv=262611 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=143616 expected_recv=255497 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=143616 expected_recv=257810 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=143616 expected_recv=262165 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=143616 expected_recv=262611 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=259072 slack_slots=28160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=143616 expected_recv=257810 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=264832 slack_slots=22400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=259456 slack_slots=27776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=143616 expected_recv=261099 expected_overflow=0 exact_recv_per_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=143616 expected_recv=263282 expected_overflow=0 exact_recv_per_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=143616 expected_recv=261099 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=143616 expected_recv=252857 expected_overflow=0 exact_recv_per_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=143616 expected_recv=260844 expected_overflow=0 exact_recv_per_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=143616 expected_recv=249733 expected_overflow=0 exact_recv_per_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=143616 expected_recv=263282 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=143616 expected_recv=252857 actual_valid_recv=255872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=143616 expected_recv=260844 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=143616 expected_recv=249733 actual_valid_recv=252160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=255872 slack_slots=31360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=252160 slack_slots=35072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=265984 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=143616 expected_recv=261969 expected_overflow=0 exact_recv_per_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=143616 expected_recv=264361 expected_overflow=0 exact_recv_per_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=143616 expected_recv=262023 expected_overflow=0 exact_recv_per_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=143616 expected_recv=261969 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=143616 expected_recv=264361 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=143616 expected_recv=262023 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=143616 expected_recv=254711 expected_overflow=0 exact_recv_per_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=143616 expected_recv=254711 actual_valid_recv=256640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=256640 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=143616 expected_recv=260618 expected_overflow=0 exact_recv_per_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=143616 expected_recv=260618 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=143616 expected_recv=257992 expected_overflow=0 exact_recv_per_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=262144 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=143616 expected_recv=257992 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=143616 expected_recv=259755 expected_overflow=0 exact_recv_per_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=260352 slack_slots=26880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=143616 expected_recv=259755 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264395 expected_overflow=0 exact_recv_per_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=143616 expected_recv=259359 expected_overflow=0 exact_recv_per_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=143616 expected_recv=249167 expected_overflow=0 exact_recv_per_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264395 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=143616 expected_recv=259359 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=143616 expected_recv=249167 actual_valid_recv=251904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=262016 slack_slots=25216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=251904 slack_slots=35328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=143616 expected_recv=262698 expected_overflow=0 exact_recv_per_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=143616 expected_recv=262698 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264832 slack_slots=22400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=143616 expected_recv=262003 expected_overflow=0 exact_recv_per_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=143616 expected_recv=260614 expected_overflow=0 exact_recv_per_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=143616 expected_recv=256427 expected_overflow=0 exact_recv_per_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=143616 expected_recv=262003 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=143616 expected_recv=260614 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=143616 expected_recv=262367 expected_overflow=0 exact_recv_per_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=143616 expected_recv=256427 actual_valid_recv=258432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264832 slack_slots=22400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=143616 expected_recv=262367 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=143616 expected_recv=262449 expected_overflow=0 exact_recv_per_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=258432 slack_slots=28800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264960 slack_slots=22272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=143616 expected_recv=262449 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=264576 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=143616 expected_recv=260264 expected_overflow=0 exact_recv_per_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=143616 expected_recv=259843 expected_overflow=0 exact_recv_per_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=143616 expected_recv=260377 expected_overflow=0 exact_recv_per_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=143616 expected_recv=260264 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=143616 expected_recv=259681 expected_overflow=0 exact_recv_per_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=143616 expected_recv=260377 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=143616 expected_recv=259843 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263424 slack_slots=23808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=143616 expected_recv=259681 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=143616 expected_recv=264000 expected_overflow=0 exact_recv_per_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=143616 expected_recv=257041 expected_overflow=0 exact_recv_per_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=143616 expected_recv=256671 expected_overflow=0 exact_recv_per_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=143616 expected_recv=264000 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=143616 expected_recv=257041 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=143616 expected_recv=256671 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266112 slack_slots=21120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=143616 expected_recv=259205 expected_overflow=0 exact_recv_per_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=143616 expected_recv=259205 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=143616 expected_recv=261917 expected_overflow=0 exact_recv_per_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=143616 expected_recv=261917 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=143616 expected_recv=252200 expected_overflow=0 exact_recv_per_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=143616 expected_recv=252200 actual_valid_recv=254336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=143616 expected_recv=259074 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=254336 slack_slots=32896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=143616 expected_recv=259074 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=143616 expected_recv=259759 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=143616 expected_recv=259759 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=143616 expected_recv=259587 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=143616 expected_recv=259668 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=143616 expected_recv=259587 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=143616 expected_recv=259668 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=261760 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=143616 expected_recv=257679 expected_overflow=0 exact_recv_per_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=143616 expected_recv=263542 expected_overflow=0 exact_recv_per_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=143616 expected_recv=257679 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=143616 expected_recv=263542 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=259712 slack_slots=27520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=265344 slack_slots=21888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=143616 expected_recv=252100 expected_overflow=0 exact_recv_per_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=143616 expected_recv=257325 expected_overflow=0 exact_recv_per_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=143616 expected_recv=252100 actual_valid_recv=255232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=255232 slack_slots=32000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=143616 expected_recv=257325 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=268288 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=259584 slack_slots=27648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=268288 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=143616 expected_recv=251230 expected_overflow=0 exact_recv_per_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264762 expected_overflow=0 exact_recv_per_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=143616 expected_recv=258563 expected_overflow=0 exact_recv_per_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=143616 expected_recv=251230 actual_valid_recv=253952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264762 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=143616 expected_recv=261631 expected_overflow=0 exact_recv_per_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=143616 expected_recv=258563 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=253952 slack_slots=33280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=143616 expected_recv=261631 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=143616 expected_recv=255687 expected_overflow=0 exact_recv_per_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=263296 slack_slots=23936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=143616 expected_recv=255687 actual_valid_recv=258432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=258432 slack_slots=28800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=143616 expected_recv=262540 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=143616 expected_recv=262207 expected_overflow=0 exact_recv_per_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=143616 expected_recv=262540 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=143616 expected_recv=262207 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=143616 expected_recv=259397 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=143616 expected_recv=257159 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=143616 expected_recv=259397 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=143616 expected_recv=257159 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=259712 slack_slots=27520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[153571, 139480, 125978, 96028, 139990, 109357, 130624, 153548] avg=131072.0000 max=153571 max_rank=0 min=96028 min_rank=3 imbalance_ratio=1.171654 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=143616 expected_recv=258990 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[153571, 139480, 125978, 96028, 139990, 109357, 130624, 153548] avg=131072.0000 max=153571 max_rank=0 min=96028 min_rank=3 imbalance_ratio=1.171654 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=143616 expected_recv=258990 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[153571, 139480, 125978, 96028, 139990, 109357, 130624, 153548] avg=131072.0000 max=153571 max_rank=0 min=96028 min_rank=3 imbalance_ratio=1.171654 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[153571, 139480, 125978, 96028, 139990, 109357, 130624, 153548] avg=131072.0000 max=153571 max_rank=0 min=96028 min_rank=3 imbalance_ratio=1.171654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] avg=131072.0000 max=132384 max_rank=4 min=125978 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21187 max_load_reduction_pct=0.137962 moved_to_replicas=115827 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[153571, 139480, 125978, 96028, 139990, 109357, 130624, 153548] avg=131072.0000 max=153571 max_rank=0 min=96028 min_rank=3 imbalance_ratio=1.171654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] avg=131072.0000 max=132384 max_rank=4 min=125978 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21187 max_load_reduction_pct=0.137962 moved_to_replicas=115827 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] avg=131072.0000 max=132384 max_rank=4 min=125978 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21187 max_load_reduction_pct=0.137962 moved_to_replicas=115827 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] avg=131072.0000 max=132384 max_rank=4 min=125978 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21187 max_load_reduction_pct=0.137962 moved_to_replicas=115827 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[153571, 139480, 125978, 96028, 139990, 109357, 130624, 153548] avg=131072.0000 max=153571 max_rank=0 min=96028 min_rank=3 imbalance_ratio=1.171654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] avg=131072.0000 max=132384 max_rank=4 min=125978 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21187 max_load_reduction_pct=0.137962 moved_to_replicas=115827 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[67494, 126448, 174977, 186825, 106767, 107774, 182837, 95454] avg=131072.0000 max=186825 max_rank=3 min=67494 min_rank=0 imbalance_ratio=1.425362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[153571, 139480, 125978, 96028, 139990, 109357, 130624, 153548] avg=131072.0000 max=153571 max_rank=0 min=96028 min_rank=3 imbalance_ratio=1.171654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[153571, 139480, 125978, 96028, 139990, 109357, 130624, 153548] avg=131072.0000 max=153571 max_rank=0 min=96028 min_rank=3 imbalance_ratio=1.171654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[67494, 126448, 174977, 186825, 106767, 107774, 182837, 95454] avg=131072.0000 max=186825 max_rank=3 min=67494 min_rank=0 imbalance_ratio=1.425362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[67494, 126448, 174977, 186825, 106767, 107774, 182837, 95454] avg=131072.0000 max=186825 max_rank=3 min=67494 min_rank=0 imbalance_ratio=1.425362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] avg=131072.0000 max=132384 max_rank=4 min=125978 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21187 max_load_reduction_pct=0.137962 moved_to_replicas=115827 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] avg=131072.0000 max=132385 max_rank=3 min=126448 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=54440 max_load_reduction_pct=0.291396 moved_to_replicas=298295 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[67494, 126448, 174977, 186825, 106767, 107774, 182837, 95454] avg=131072.0000 max=186825 max_rank=3 min=67494 min_rank=0 imbalance_ratio=1.425362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] avg=131072.0000 max=132384 max_rank=4 min=125978 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21187 max_load_reduction_pct=0.137962 moved_to_replicas=115827 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[67494, 126448, 174977, 186825, 106767, 107774, 182837, 95454] avg=131072.0000 max=186825 max_rank=3 min=67494 min_rank=0 imbalance_ratio=1.425362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] avg=131072.0000 max=132385 max_rank=3 min=126448 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=54440 max_load_reduction_pct=0.291396 moved_to_replicas=298295 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125978, 132225, 132384, 130217, 130624, 132382] avg=131072.0000 max=132384 max_rank=4 min=125978 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=21187 max_load_reduction_pct=0.137962 moved_to_replicas=115827 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] avg=131072.0000 max=132385 max_rank=3 min=126448 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=54440 max_load_reduction_pct=0.291396 moved_to_replicas=298295 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] avg=131072.0000 max=132385 max_rank=3 min=126448 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=54440 max_load_reduction_pct=0.291396 moved_to_replicas=298295 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[103428, 223646, 131002, 236469, 121154, 137412, 53857, 41608] avg=131072.0000 max=236469 max_rank=3 min=41608 min_rank=7 imbalance_ratio=1.804115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] avg=131072.0000 max=132385 max_rank=3 min=126448 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=54440 max_load_reduction_pct=0.291396 moved_to_replicas=298295 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[67494, 126448, 174977, 186825, 106767, 107774, 182837, 95454] avg=131072.0000 max=186825 max_rank=3 min=67494 min_rank=0 imbalance_ratio=1.425362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[67494, 126448, 174977, 186825, 106767, 107774, 182837, 95454] avg=131072.0000 max=186825 max_rank=3 min=67494 min_rank=0 imbalance_ratio=1.425362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[103428, 223646, 131002, 236469, 121154, 137412, 53857, 41608] avg=131072.0000 max=236469 max_rank=3 min=41608 min_rank=7 imbalance_ratio=1.804115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[67494, 126448, 174977, 186825, 106767, 107774, 182837, 95454] avg=131072.0000 max=186825 max_rank=3 min=67494 min_rank=0 imbalance_ratio=1.425362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] avg=131072.0000 max=132385 max_rank=5 min=125426 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=104084 max_load_reduction_pct=0.440159 moved_to_replicas=410149 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[103428, 223646, 131002, 236469, 121154, 137412, 53857, 41608] avg=131072.0000 max=236469 max_rank=3 min=41608 min_rank=7 imbalance_ratio=1.804115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] avg=131072.0000 max=132385 max_rank=3 min=126448 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=54440 max_load_reduction_pct=0.291396 moved_to_replicas=298295 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[103428, 223646, 131002, 236469, 121154, 137412, 53857, 41608] avg=131072.0000 max=236469 max_rank=3 min=41608 min_rank=7 imbalance_ratio=1.804115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] avg=131072.0000 max=132385 max_rank=3 min=126448 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=54440 max_load_reduction_pct=0.291396 moved_to_replicas=298295 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] avg=131072.0000 max=132385 max_rank=5 min=125426 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=104084 max_load_reduction_pct=0.440159 moved_to_replicas=410149 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[103428, 223646, 131002, 236469, 121154, 137412, 53857, 41608] avg=131072.0000 max=236469 max_rank=3 min=41608 min_rank=7 imbalance_ratio=1.804115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[132383, 126448, 132382, 132385, 132384, 132383, 132383, 127828] avg=131072.0000 max=132385 max_rank=3 min=126448 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=54440 max_load_reduction_pct=0.291396 moved_to_replicas=298295 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] avg=131072.0000 max=132385 max_rank=5 min=125426 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=104084 max_load_reduction_pct=0.440159 moved_to_replicas=410149 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] avg=131072.0000 max=132385 max_rank=5 min=125426 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=104084 max_load_reduction_pct=0.440159 moved_to_replicas=410149 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[156835, 128462, 135891, 177622, 101376, 157103, 96735, 94552] avg=131072.0000 max=177622 max_rank=3 min=94552 min_rank=7 imbalance_ratio=1.355148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[103428, 223646, 131002, 236469, 121154, 137412, 53857, 41608] avg=131072.0000 max=236469 max_rank=3 min=41608 min_rank=7 imbalance_ratio=1.804115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] avg=131072.0000 max=132385 max_rank=5 min=125426 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=104084 max_load_reduction_pct=0.440159 moved_to_replicas=410149 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[156835, 128462, 135891, 177622, 101376, 157103, 96735, 94552] avg=131072.0000 max=177622 max_rank=3 min=94552 min_rank=7 imbalance_ratio=1.355148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[103428, 223646, 131002, 236469, 121154, 137412, 53857, 41608] avg=131072.0000 max=236469 max_rank=3 min=41608 min_rank=7 imbalance_ratio=1.804115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] avg=131072.0000 max=132384 max_rank=0 min=127372 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=45238 max_load_reduction_pct=0.254687 moved_to_replicas=195636 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[103428, 223646, 131002, 236469, 121154, 137412, 53857, 41608] avg=131072.0000 max=236469 max_rank=3 min=41608 min_rank=7 imbalance_ratio=1.804115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[156835, 128462, 135891, 177622, 101376, 157103, 96735, 94552] avg=131072.0000 max=177622 max_rank=3 min=94552 min_rank=7 imbalance_ratio=1.355148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] avg=131072.0000 max=132385 max_rank=5 min=125426 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=104084 max_load_reduction_pct=0.440159 moved_to_replicas=410149 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[156835, 128462, 135891, 177622, 101376, 157103, 96735, 94552] avg=131072.0000 max=177622 max_rank=3 min=94552 min_rank=7 imbalance_ratio=1.355148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] avg=131072.0000 max=132384 max_rank=0 min=127372 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=45238 max_load_reduction_pct=0.254687 moved_to_replicas=195636 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] avg=131072.0000 max=132385 max_rank=5 min=125426 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=104084 max_load_reduction_pct=0.440159 moved_to_replicas=410149 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[156835, 128462, 135891, 177622, 101376, 157103, 96735, 94552] avg=131072.0000 max=177622 max_rank=3 min=94552 min_rank=7 imbalance_ratio=1.355148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[132383, 132383, 131002, 132383, 125426, 132385, 130230, 132384] avg=131072.0000 max=132385 max_rank=5 min=125426 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=104084 max_load_reduction_pct=0.440159 moved_to_replicas=410149 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] avg=131072.0000 max=132384 max_rank=0 min=127372 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=45238 max_load_reduction_pct=0.254687 moved_to_replicas=195636 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[137999, 100520, 172533, 65900, 131034, 131016, 164015, 145559] avg=131072.0000 max=172533 max_rank=2 min=65900 min_rank=3 imbalance_ratio=1.316322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] avg=131072.0000 max=132384 max_rank=0 min=127372 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=45238 max_load_reduction_pct=0.254687 moved_to_replicas=195636 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[156835, 128462, 135891, 177622, 101376, 157103, 96735, 94552] avg=131072.0000 max=177622 max_rank=3 min=94552 min_rank=7 imbalance_ratio=1.355148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] avg=131072.0000 max=132384 max_rank=0 min=127372 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=45238 max_load_reduction_pct=0.254687 moved_to_replicas=195636 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[137999, 100520, 172533, 65900, 131034, 131016, 164015, 145559] avg=131072.0000 max=172533 max_rank=2 min=65900 min_rank=3 imbalance_ratio=1.316322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[156835, 128462, 135891, 177622, 101376, 157103, 96735, 94552] avg=131072.0000 max=177622 max_rank=3 min=94552 min_rank=7 imbalance_ratio=1.355148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] avg=131072.0000 max=132385 max_rank=0 min=124838 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=40148 max_load_reduction_pct=0.232698 moved_to_replicas=181527 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[137999, 100520, 172533, 65900, 131034, 131016, 164015, 145559] avg=131072.0000 max=172533 max_rank=2 min=65900 min_rank=3 imbalance_ratio=1.316322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] avg=131072.0000 max=132384 max_rank=0 min=127372 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=45238 max_load_reduction_pct=0.254687 moved_to_replicas=195636 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[156835, 128462, 135891, 177622, 101376, 157103, 96735, 94552] avg=131072.0000 max=177622 max_rank=3 min=94552 min_rank=7 imbalance_ratio=1.355148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] avg=131072.0000 max=132385 max_rank=0 min=124838 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=40148 max_load_reduction_pct=0.232698 moved_to_replicas=181527 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[137999, 100520, 172533, 65900, 131034, 131016, 164015, 145559] avg=131072.0000 max=172533 max_rank=2 min=65900 min_rank=3 imbalance_ratio=1.316322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] avg=131072.0000 max=132384 max_rank=0 min=127372 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=45238 max_load_reduction_pct=0.254687 moved_to_replicas=195636 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[137999, 100520, 172533, 65900, 131034, 131016, 164015, 145559] avg=131072.0000 max=172533 max_rank=2 min=65900 min_rank=3 imbalance_ratio=1.316322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] avg=131072.0000 max=132385 max_rank=0 min=124838 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=40148 max_load_reduction_pct=0.232698 moved_to_replicas=181527 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[132384, 128462, 132383, 132384, 127372, 132384, 132381, 130826] avg=131072.0000 max=132384 max_rank=0 min=127372 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=45238 max_load_reduction_pct=0.254687 moved_to_replicas=195636 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[91376, 98308, 106034, 163731, 168604, 261484, 118501, 40538] avg=131072.0000 max=261484 max_rank=5 min=40538 min_rank=7 imbalance_ratio=1.994965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[137999, 100520, 172533, 65900, 131034, 131016, 164015, 145559] avg=131072.0000 max=172533 max_rank=2 min=65900 min_rank=3 imbalance_ratio=1.316322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] avg=131072.0000 max=132385 max_rank=0 min=124838 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=40148 max_load_reduction_pct=0.232698 moved_to_replicas=181527 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] avg=131072.0000 max=132385 max_rank=0 min=124838 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=40148 max_load_reduction_pct=0.232698 moved_to_replicas=181527 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[91376, 98308, 106034, 163731, 168604, 261484, 118501, 40538] avg=131072.0000 max=261484 max_rank=5 min=40538 min_rank=7 imbalance_ratio=1.994965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[137999, 100520, 172533, 65900, 131034, 131016, 164015, 145559] avg=131072.0000 max=172533 max_rank=2 min=65900 min_rank=3 imbalance_ratio=1.316322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125167 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=129099 max_load_reduction_pct=0.493717 moved_to_replicas=393138 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] avg=131072.0000 max=132385 max_rank=0 min=124838 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=40148 max_load_reduction_pct=0.232698 moved_to_replicas=181527 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[91376, 98308, 106034, 163731, 168604, 261484, 118501, 40538] avg=131072.0000 max=261484 max_rank=5 min=40538 min_rank=7 imbalance_ratio=1.994965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[137999, 100520, 172533, 65900, 131034, 131016, 164015, 145559] avg=131072.0000 max=172533 max_rank=2 min=65900 min_rank=3 imbalance_ratio=1.316322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125167 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=129099 max_load_reduction_pct=0.493717 moved_to_replicas=393138 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[91376, 98308, 106034, 163731, 168604, 261484, 118501, 40538] avg=131072.0000 max=261484 max_rank=5 min=40538 min_rank=7 imbalance_ratio=1.994965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] avg=131072.0000 max=132385 max_rank=0 min=124838 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=40148 max_load_reduction_pct=0.232698 moved_to_replicas=181527 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[91376, 98308, 106034, 163731, 168604, 261484, 118501, 40538] avg=131072.0000 max=261484 max_rank=5 min=40538 min_rank=7 imbalance_ratio=1.994965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125167 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=129099 max_load_reduction_pct=0.493717 moved_to_replicas=393138 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[132385, 132152, 132383, 124838, 131034, 131016, 132383, 132385] avg=131072.0000 max=132385 max_rank=0 min=124838 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=40148 max_load_reduction_pct=0.232698 moved_to_replicas=181527 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[75926, 98342, 196520, 161918, 100256, 164139, 187173, 64302] avg=131072.0000 max=196520 max_rank=2 min=64302 min_rank=7 imbalance_ratio=1.499329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[91376, 98308, 106034, 163731, 168604, 261484, 118501, 40538] avg=131072.0000 max=261484 max_rank=5 min=40538 min_rank=7 imbalance_ratio=1.994965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125167 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=129099 max_load_reduction_pct=0.493717 moved_to_replicas=393138 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125167 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=129099 max_load_reduction_pct=0.493717 moved_to_replicas=393138 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[75926, 98342, 196520, 161918, 100256, 164139, 187173, 64302] avg=131072.0000 max=196520 max_rank=2 min=64302 min_rank=7 imbalance_ratio=1.499329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] avg=131072.0000 max=132384 max_rank=0 min=124815 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64136 max_load_reduction_pct=0.326359 moved_to_replicas=354843 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[91376, 98308, 106034, 163731, 168604, 261484, 118501, 40538] avg=131072.0000 max=261484 max_rank=5 min=40538 min_rank=7 imbalance_ratio=1.994965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125167 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=129099 max_load_reduction_pct=0.493717 moved_to_replicas=393138 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[75926, 98342, 196520, 161918, 100256, 164139, 187173, 64302] avg=131072.0000 max=196520 max_rank=2 min=64302 min_rank=7 imbalance_ratio=1.499329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[91376, 98308, 106034, 163731, 168604, 261484, 118501, 40538] avg=131072.0000 max=261484 max_rank=5 min=40538 min_rank=7 imbalance_ratio=1.994965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] avg=131072.0000 max=132384 max_rank=0 min=124815 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64136 max_load_reduction_pct=0.326359 moved_to_replicas=354843 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125167 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=129099 max_load_reduction_pct=0.493717 moved_to_replicas=393138 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] avg=131072.0000 max=132384 max_rank=0 min=124815 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64136 max_load_reduction_pct=0.326359 moved_to_replicas=354843 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[75926, 98342, 196520, 161918, 100256, 164139, 187173, 64302] avg=131072.0000 max=196520 max_rank=2 min=64302 min_rank=7 imbalance_ratio=1.499329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[75926, 98342, 196520, 161918, 100256, 164139, 187173, 64302] avg=131072.0000 max=196520 max_rank=2 min=64302 min_rank=7 imbalance_ratio=1.499329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[65938, 76188, 210413, 98284, 98278, 131174, 73459, 294842] avg=131072.0000 max=294842 max_rank=7 min=65938 min_rank=0 imbalance_ratio=2.249466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[125167, 129111, 132381, 132383, 132383, 132383, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125167 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=129099 max_load_reduction_pct=0.493717 moved_to_replicas=393138 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[75926, 98342, 196520, 161918, 100256, 164139, 187173, 64302] avg=131072.0000 max=196520 max_rank=2 min=64302 min_rank=7 imbalance_ratio=1.499329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[65938, 76188, 210413, 98284, 98278, 131174, 73459, 294842] avg=131072.0000 max=294842 max_rank=7 min=65938 min_rank=0 imbalance_ratio=2.249466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] avg=131072.0000 max=132384 max_rank=0 min=124815 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64136 max_load_reduction_pct=0.326359 moved_to_replicas=354843 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=128060 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=162457 max_load_reduction_pct=0.550997 moved_to_replicas=475606 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] avg=131072.0000 max=132384 max_rank=0 min=124815 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64136 max_load_reduction_pct=0.326359 moved_to_replicas=354843 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[75926, 98342, 196520, 161918, 100256, 164139, 187173, 64302] avg=131072.0000 max=196520 max_rank=2 min=64302 min_rank=7 imbalance_ratio=1.499329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[65938, 76188, 210413, 98284, 98278, 131174, 73459, 294842] avg=131072.0000 max=294842 max_rank=7 min=65938 min_rank=0 imbalance_ratio=2.249466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] avg=131072.0000 max=132384 max_rank=0 min=124815 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64136 max_load_reduction_pct=0.326359 moved_to_replicas=354843 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[75926, 98342, 196520, 161918, 100256, 164139, 187173, 64302] avg=131072.0000 max=196520 max_rank=2 min=64302 min_rank=7 imbalance_ratio=1.499329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=128060 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=162457 max_load_reduction_pct=0.550997 moved_to_replicas=475606 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] avg=131072.0000 max=132384 max_rank=0 min=124815 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64136 max_load_reduction_pct=0.326359 moved_to_replicas=354843 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=128060 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=162457 max_load_reduction_pct=0.550997 moved_to_replicas=475606 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[98541, 129336, 121487, 129852, 130047, 106462, 158559, 174292] avg=131072.0000 max=174292 max_rank=7 min=98541 min_rank=0 imbalance_ratio=1.329742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[65938, 76188, 210413, 98284, 98278, 131174, 73459, 294842] avg=131072.0000 max=294842 max_rank=7 min=65938 min_rank=0 imbalance_ratio=2.249466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[65938, 76188, 210413, 98284, 98278, 131174, 73459, 294842] avg=131072.0000 max=294842 max_rank=7 min=65938 min_rank=0 imbalance_ratio=2.249466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[65938, 76188, 210413, 98284, 98278, 131174, 73459, 294842] avg=131072.0000 max=294842 max_rank=7 min=65938 min_rank=0 imbalance_ratio=2.249466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[132384, 124815, 132383, 132383, 132012, 132383, 132384, 129832] avg=131072.0000 max=132384 max_rank=0 min=124815 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64136 max_load_reduction_pct=0.326359 moved_to_replicas=354843 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[98541, 129336, 121487, 129852, 130047, 106462, 158559, 174292] avg=131072.0000 max=174292 max_rank=7 min=98541 min_rank=0 imbalance_ratio=1.329742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] avg=131072.0000 max=132384 max_rank=5 min=129336 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=41908 max_load_reduction_pct=0.240447 moved_to_replicas=130215 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[65938, 76188, 210413, 98284, 98278, 131174, 73459, 294842] avg=131072.0000 max=294842 max_rank=7 min=65938 min_rank=0 imbalance_ratio=2.249466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=128060 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=162457 max_load_reduction_pct=0.550997 moved_to_replicas=475606 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[98541, 129336, 121487, 129852, 130047, 106462, 158559, 174292] avg=131072.0000 max=174292 max_rank=7 min=98541 min_rank=0 imbalance_ratio=1.329742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=128060 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=162457 max_load_reduction_pct=0.550997 moved_to_replicas=475606 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=128060 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=162457 max_load_reduction_pct=0.550997 moved_to_replicas=475606 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] avg=131072.0000 max=132384 max_rank=5 min=129336 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=41908 max_load_reduction_pct=0.240447 moved_to_replicas=130215 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[65938, 76188, 210413, 98284, 98278, 131174, 73459, 294842] avg=131072.0000 max=294842 max_rank=7 min=65938 min_rank=0 imbalance_ratio=2.249466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=128060 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=162457 max_load_reduction_pct=0.550997 moved_to_replicas=475606 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] avg=131072.0000 max=132384 max_rank=5 min=129336 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=41908 max_load_reduction_pct=0.240447 moved_to_replicas=130215 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[135672, 91900, 133640, 98389, 33434, 163953, 162387, 229201] avg=131072.0000 max=229201 max_rank=7 min=33434 min_rank=4 imbalance_ratio=1.748665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[98541, 129336, 121487, 129852, 130047, 106462, 158559, 174292] avg=131072.0000 max=174292 max_rank=7 min=98541 min_rank=0 imbalance_ratio=1.329742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[131469, 128060, 132381, 129684, 131040, 131174, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=128060 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=162457 max_load_reduction_pct=0.550997 moved_to_replicas=475606 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[98541, 129336, 121487, 129852, 130047, 106462, 158559, 174292] avg=131072.0000 max=174292 max_rank=7 min=98541 min_rank=0 imbalance_ratio=1.329742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[98541, 129336, 121487, 129852, 130047, 106462, 158559, 174292] avg=131072.0000 max=174292 max_rank=7 min=98541 min_rank=0 imbalance_ratio=1.329742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[135672, 91900, 133640, 98389, 33434, 163953, 162387, 229201] avg=131072.0000 max=229201 max_rank=7 min=33434 min_rank=4 imbalance_ratio=1.748665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127734 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96818 max_load_reduction_pct=0.422415 moved_to_replicas=327907 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[98541, 129336, 121487, 129852, 130047, 106462, 158559, 174292] avg=131072.0000 max=174292 max_rank=7 min=98541 min_rank=0 imbalance_ratio=1.329742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] avg=131072.0000 max=132384 max_rank=5 min=129336 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=41908 max_load_reduction_pct=0.240447 moved_to_replicas=130215 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[135672, 91900, 133640, 98389, 33434, 163953, 162387, 229201] avg=131072.0000 max=229201 max_rank=7 min=33434 min_rank=4 imbalance_ratio=1.748665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] avg=131072.0000 max=132384 max_rank=5 min=129336 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=41908 max_load_reduction_pct=0.240447 moved_to_replicas=130215 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] avg=131072.0000 max=132384 max_rank=5 min=129336 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=41908 max_load_reduction_pct=0.240447 moved_to_replicas=130215 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127734 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96818 max_load_reduction_pct=0.422415 moved_to_replicas=327907 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[98541, 129336, 121487, 129852, 130047, 106462, 158559, 174292] avg=131072.0000 max=174292 max_rank=7 min=98541 min_rank=0 imbalance_ratio=1.329742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] avg=131072.0000 max=132384 max_rank=5 min=129336 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=41908 max_load_reduction_pct=0.240447 moved_to_replicas=130215 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127734 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96818 max_load_reduction_pct=0.422415 moved_to_replicas=327907 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[64061, 220375, 65293, 164048, 68770, 200415, 98218, 167396] avg=131072.0000 max=220375 max_rank=1 min=64061 min_rank=0 imbalance_ratio=1.681328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[135672, 91900, 133640, 98389, 33434, 163953, 162387, 229201] avg=131072.0000 max=229201 max_rank=7 min=33434 min_rank=4 imbalance_ratio=1.748665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[131307, 129336, 130883, 129852, 130047, 132384, 132383, 132384] avg=131072.0000 max=132384 max_rank=5 min=129336 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=41908 max_load_reduction_pct=0.240447 moved_to_replicas=130215 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[135672, 91900, 133640, 98389, 33434, 163953, 162387, 229201] avg=131072.0000 max=229201 max_rank=7 min=33434 min_rank=4 imbalance_ratio=1.748665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[135672, 91900, 133640, 98389, 33434, 163953, 162387, 229201] avg=131072.0000 max=229201 max_rank=7 min=33434 min_rank=4 imbalance_ratio=1.748665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[64061, 220375, 65293, 164048, 68770, 200415, 98218, 167396] avg=131072.0000 max=220375 max_rank=1 min=64061 min_rank=0 imbalance_ratio=1.681328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[135672, 91900, 133640, 98389, 33434, 163953, 162387, 229201] avg=131072.0000 max=229201 max_rank=7 min=33434 min_rank=4 imbalance_ratio=1.748665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] avg=131072.0000 max=132383 max_rank=1 min=126085 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=87992 max_load_reduction_pct=0.399283 moved_to_replicas=447562 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127734 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96818 max_load_reduction_pct=0.422415 moved_to_replicas=327907 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[64061, 220375, 65293, 164048, 68770, 200415, 98218, 167396] avg=131072.0000 max=220375 max_rank=1 min=64061 min_rank=0 imbalance_ratio=1.681328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127734 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96818 max_load_reduction_pct=0.422415 moved_to_replicas=327907 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127734 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96818 max_load_reduction_pct=0.422415 moved_to_replicas=327907 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] avg=131072.0000 max=132383 max_rank=1 min=126085 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=87992 max_load_reduction_pct=0.399283 moved_to_replicas=447562 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[135672, 91900, 133640, 98389, 33434, 163953, 162387, 229201] avg=131072.0000 max=229201 max_rank=7 min=33434 min_rank=4 imbalance_ratio=1.748665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127734 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96818 max_load_reduction_pct=0.422415 moved_to_replicas=327907 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] avg=131072.0000 max=132383 max_rank=1 min=126085 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=87992 max_load_reduction_pct=0.399283 moved_to_replicas=447562 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[196833, 163671, 98583, 163401, 98153, 163827, 98415, 65693] avg=131072.0000 max=196833 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.501717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[64061, 220375, 65293, 164048, 68770, 200415, 98218, 167396] avg=131072.0000 max=220375 max_rank=1 min=64061 min_rank=0 imbalance_ratio=1.681328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[132383, 127734, 132383, 129959, 128968, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127734 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=96818 max_load_reduction_pct=0.422415 moved_to_replicas=327907 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[64061, 220375, 65293, 164048, 68770, 200415, 98218, 167396] avg=131072.0000 max=220375 max_rank=1 min=64061 min_rank=0 imbalance_ratio=1.681328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[64061, 220375, 65293, 164048, 68770, 200415, 98218, 167396] avg=131072.0000 max=220375 max_rank=1 min=64061 min_rank=0 imbalance_ratio=1.681328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[196833, 163671, 98583, 163401, 98153, 163827, 98415, 65693] avg=131072.0000 max=196833 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.501717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[64061, 220375, 65293, 164048, 68770, 200415, 98218, 167396] avg=131072.0000 max=220375 max_rank=1 min=64061 min_rank=0 imbalance_ratio=1.681328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] avg=131072.0000 max=132383 max_rank=0 min=129601 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64450 max_load_reduction_pct=0.327435 moved_to_replicas=316100 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] avg=131072.0000 max=132383 max_rank=1 min=126085 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=87992 max_load_reduction_pct=0.399283 moved_to_replicas=447562 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[196833, 163671, 98583, 163401, 98153, 163827, 98415, 65693] avg=131072.0000 max=196833 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.501717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] avg=131072.0000 max=132383 max_rank=1 min=126085 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=87992 max_load_reduction_pct=0.399283 moved_to_replicas=447562 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] avg=131072.0000 max=132383 max_rank=1 min=126085 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=87992 max_load_reduction_pct=0.399283 moved_to_replicas=447562 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] avg=131072.0000 max=132383 max_rank=0 min=129601 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64450 max_load_reduction_pct=0.327435 moved_to_replicas=316100 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[64061, 220375, 65293, 164048, 68770, 200415, 98218, 167396] avg=131072.0000 max=220375 max_rank=1 min=64061 min_rank=0 imbalance_ratio=1.681328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] avg=131072.0000 max=132383 max_rank=1 min=126085 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=87992 max_load_reduction_pct=0.399283 moved_to_replicas=447562 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] avg=131072.0000 max=132383 max_rank=0 min=129601 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64450 max_load_reduction_pct=0.327435 moved_to_replicas=316100 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[131700, 229338, 11019, 163749, 117167, 133946, 163712, 97945] avg=131072.0000 max=229338 max_rank=1 min=11019 min_rank=2 imbalance_ratio=1.749710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[196833, 163671, 98583, 163401, 98153, 163827, 98415, 65693] avg=131072.0000 max=196833 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.501717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[129592, 132383, 132383, 132383, 126085, 132383, 130984, 132383] avg=131072.0000 max=132383 max_rank=1 min=126085 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=87992 max_load_reduction_pct=0.399283 moved_to_replicas=447562 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[196833, 163671, 98583, 163401, 98153, 163827, 98415, 65693] avg=131072.0000 max=196833 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.501717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[196833, 163671, 98583, 163401, 98153, 163827, 98415, 65693] avg=131072.0000 max=196833 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.501717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] avg=131072.0000 max=132383 max_rank=1 min=125650 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96955 max_load_reduction_pct=0.422760 moved_to_replicas=323298 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[131700, 229338, 11019, 163749, 117167, 133946, 163712, 97945] avg=131072.0000 max=229338 max_rank=1 min=11019 min_rank=2 imbalance_ratio=1.749710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[196833, 163671, 98583, 163401, 98153, 163827, 98415, 65693] avg=131072.0000 max=196833 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.501717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] avg=131072.0000 max=132383 max_rank=0 min=129601 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64450 max_load_reduction_pct=0.327435 moved_to_replicas=316100 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[131700, 229338, 11019, 163749, 117167, 133946, 163712, 97945] avg=131072.0000 max=229338 max_rank=1 min=11019 min_rank=2 imbalance_ratio=1.749710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] avg=131072.0000 max=132383 max_rank=0 min=129601 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64450 max_load_reduction_pct=0.327435 moved_to_replicas=316100 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] avg=131072.0000 max=132383 max_rank=0 min=129601 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64450 max_load_reduction_pct=0.327435 moved_to_replicas=316100 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] avg=131072.0000 max=132383 max_rank=1 min=125650 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96955 max_load_reduction_pct=0.422760 moved_to_replicas=323298 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] avg=131072.0000 max=132383 max_rank=0 min=129601 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64450 max_load_reduction_pct=0.327435 moved_to_replicas=316100 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[196833, 163671, 98583, 163401, 98153, 163827, 98415, 65693] avg=131072.0000 max=196833 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.501717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] avg=131072.0000 max=132383 max_rank=1 min=125650 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96955 max_load_reduction_pct=0.422760 moved_to_replicas=323298 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[65522, 172187, 67809, 167310, 91929, 228136, 171075, 84608] avg=131072.0000 max=228136 max_rank=5 min=65522 min_rank=0 imbalance_ratio=1.740540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[131700, 229338, 11019, 163749, 117167, 133946, 163712, 97945] avg=131072.0000 max=229338 max_rank=1 min=11019 min_rank=2 imbalance_ratio=1.749710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129601, 132383, 129837, 132383, 129859, 129747] avg=131072.0000 max=132383 max_rank=0 min=129601 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64450 max_load_reduction_pct=0.327435 moved_to_replicas=316100 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[131700, 229338, 11019, 163749, 117167, 133946, 163712, 97945] avg=131072.0000 max=229338 max_rank=1 min=11019 min_rank=2 imbalance_ratio=1.749710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[131700, 229338, 11019, 163749, 117167, 133946, 163712, 97945] avg=131072.0000 max=229338 max_rank=1 min=11019 min_rank=2 imbalance_ratio=1.749710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] avg=131072.0000 max=132385 max_rank=0 min=126856 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=95751 max_load_reduction_pct=0.419710 moved_to_replicas=417110 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[65522, 172187, 67809, 167310, 91929, 228136, 171075, 84608] avg=131072.0000 max=228136 max_rank=5 min=65522 min_rank=0 imbalance_ratio=1.740540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[131700, 229338, 11019, 163749, 117167, 133946, 163712, 97945] avg=131072.0000 max=229338 max_rank=1 min=11019 min_rank=2 imbalance_ratio=1.749710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] avg=131072.0000 max=132383 max_rank=1 min=125650 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96955 max_load_reduction_pct=0.422760 moved_to_replicas=323298 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[65522, 172187, 67809, 167310, 91929, 228136, 171075, 84608] avg=131072.0000 max=228136 max_rank=5 min=65522 min_rank=0 imbalance_ratio=1.740540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] avg=131072.0000 max=132383 max_rank=1 min=125650 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96955 max_load_reduction_pct=0.422760 moved_to_replicas=323298 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] avg=131072.0000 max=132383 max_rank=1 min=125650 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96955 max_load_reduction_pct=0.422760 moved_to_replicas=323298 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] avg=131072.0000 max=132385 max_rank=0 min=126856 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=95751 max_load_reduction_pct=0.419710 moved_to_replicas=417110 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] avg=131072.0000 max=132383 max_rank=1 min=125650 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96955 max_load_reduction_pct=0.422760 moved_to_replicas=323298 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[131700, 229338, 11019, 163749, 117167, 133946, 163712, 97945] avg=131072.0000 max=229338 max_rank=1 min=11019 min_rank=2 imbalance_ratio=1.749710 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] avg=131072.0000 max=132385 max_rank=0 min=126856 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=95751 max_load_reduction_pct=0.419710 moved_to_replicas=417110 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[130868, 98326, 90219, 70102, 134990, 196650, 229333, 98088] avg=131072.0000 max=229333 max_rank=6 min=70102 min_rank=3 imbalance_ratio=1.749672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[65522, 172187, 67809, 167310, 91929, 228136, 171075, 84608] avg=131072.0000 max=228136 max_rank=5 min=65522 min_rank=0 imbalance_ratio=1.740540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[131700, 132383, 132383, 132383, 125650, 132383, 132383, 129311] avg=131072.0000 max=132383 max_rank=1 min=125650 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96955 max_load_reduction_pct=0.422760 moved_to_replicas=323298 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[65522, 172187, 67809, 167310, 91929, 228136, 171075, 84608] avg=131072.0000 max=228136 max_rank=5 min=65522 min_rank=0 imbalance_ratio=1.740540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] avg=131072.0000 max=132383 max_rank=3 min=127579 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96950 max_load_reduction_pct=0.422748 moved_to_replicas=325067 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[130868, 98326, 90219, 70102, 134990, 196650, 229333, 98088] avg=131072.0000 max=229333 max_rank=6 min=70102 min_rank=3 imbalance_ratio=1.749672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[65522, 172187, 67809, 167310, 91929, 228136, 171075, 84608] avg=131072.0000 max=228136 max_rank=5 min=65522 min_rank=0 imbalance_ratio=1.740540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] avg=131072.0000 max=132385 max_rank=0 min=126856 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=95751 max_load_reduction_pct=0.419710 moved_to_replicas=417110 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[65522, 172187, 67809, 167310, 91929, 228136, 171075, 84608] avg=131072.0000 max=228136 max_rank=5 min=65522 min_rank=0 imbalance_ratio=1.740540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[130868, 98326, 90219, 70102, 134990, 196650, 229333, 98088] avg=131072.0000 max=229333 max_rank=6 min=70102 min_rank=3 imbalance_ratio=1.749672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] avg=131072.0000 max=132385 max_rank=0 min=126856 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=95751 max_load_reduction_pct=0.419710 moved_to_replicas=417110 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] avg=131072.0000 max=132383 max_rank=3 min=127579 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96950 max_load_reduction_pct=0.422748 moved_to_replicas=325067 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] avg=131072.0000 max=132385 max_rank=0 min=126856 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=95751 max_load_reduction_pct=0.419710 moved_to_replicas=417110 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[65522, 172187, 67809, 167310, 91929, 228136, 171075, 84608] avg=131072.0000 max=228136 max_rank=5 min=65522 min_rank=0 imbalance_ratio=1.740540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] avg=131072.0000 max=132385 max_rank=0 min=126856 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=95751 max_load_reduction_pct=0.419710 moved_to_replicas=417110 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[32980, 196352, 65149, 131613, 64472, 226036, 134803, 197171] avg=131072.0000 max=226036 max_rank=5 min=32980 min_rank=0 imbalance_ratio=1.724518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] avg=131072.0000 max=132383 max_rank=3 min=127579 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96950 max_load_reduction_pct=0.422748 moved_to_replicas=325067 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[130868, 98326, 90219, 70102, 134990, 196650, 229333, 98088] avg=131072.0000 max=229333 max_rank=6 min=70102 min_rank=3 imbalance_ratio=1.749672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[132385, 132380, 132383, 132383, 126856, 132383, 132385, 127421] avg=131072.0000 max=132385 max_rank=0 min=126856 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=95751 max_load_reduction_pct=0.419710 moved_to_replicas=417110 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[130868, 98326, 90219, 70102, 134990, 196650, 229333, 98088] avg=131072.0000 max=229333 max_rank=6 min=70102 min_rank=3 imbalance_ratio=1.749672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127718 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=93652 max_load_reduction_pct=0.414323 moved_to_replicas=451238 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[32980, 196352, 65149, 131613, 64472, 226036, 134803, 197171] avg=131072.0000 max=226036 max_rank=5 min=32980 min_rank=0 imbalance_ratio=1.724518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[130868, 98326, 90219, 70102, 134990, 196650, 229333, 98088] avg=131072.0000 max=229333 max_rank=6 min=70102 min_rank=3 imbalance_ratio=1.749672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] avg=131072.0000 max=132383 max_rank=3 min=127579 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96950 max_load_reduction_pct=0.422748 moved_to_replicas=325067 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[130868, 98326, 90219, 70102, 134990, 196650, 229333, 98088] avg=131072.0000 max=229333 max_rank=6 min=70102 min_rank=3 imbalance_ratio=1.749672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[32980, 196352, 65149, 131613, 64472, 226036, 134803, 197171] avg=131072.0000 max=226036 max_rank=5 min=32980 min_rank=0 imbalance_ratio=1.724518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] avg=131072.0000 max=132383 max_rank=3 min=127579 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96950 max_load_reduction_pct=0.422748 moved_to_replicas=325067 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127718 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=93652 max_load_reduction_pct=0.414323 moved_to_replicas=451238 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] avg=131072.0000 max=132383 max_rank=3 min=127579 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96950 max_load_reduction_pct=0.422748 moved_to_replicas=325067 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[130868, 98326, 90219, 70102, 134990, 196650, 229333, 98088] avg=131072.0000 max=229333 max_rank=6 min=70102 min_rank=3 imbalance_ratio=1.749672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] avg=131072.0000 max=132383 max_rank=3 min=127579 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96950 max_load_reduction_pct=0.422748 moved_to_replicas=325067 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127718 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=93652 max_load_reduction_pct=0.414323 moved_to_replicas=451238 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[32980, 196352, 65149, 131613, 64472, 226036, 134803, 197171] avg=131072.0000 max=226036 max_rank=5 min=32980 min_rank=0 imbalance_ratio=1.724518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[130868, 131091, 127579, 132383, 132383, 132383, 132383, 129506] avg=131072.0000 max=132383 max_rank=3 min=127579 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96950 max_load_reduction_pct=0.422748 moved_to_replicas=325067 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[32980, 196352, 65149, 131613, 64472, 226036, 134803, 197171] avg=131072.0000 max=226036 max_rank=5 min=32980 min_rank=0 imbalance_ratio=1.724518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127718 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=93652 max_load_reduction_pct=0.414323 moved_to_replicas=451238 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[32980, 196352, 65149, 131613, 64472, 226036, 134803, 197171] avg=131072.0000 max=226036 max_rank=5 min=32980 min_rank=0 imbalance_ratio=1.724518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[32980, 196352, 65149, 131613, 64472, 226036, 134803, 197171] avg=131072.0000 max=226036 max_rank=5 min=32980 min_rank=0 imbalance_ratio=1.724518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127718 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=93652 max_load_reduction_pct=0.414323 moved_to_replicas=451238 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127718 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=93652 max_load_reduction_pct=0.414323 moved_to_replicas=451238 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[32980, 196352, 65149, 131613, 64472, 226036, 134803, 197171] avg=131072.0000 max=226036 max_rank=5 min=32980 min_rank=0 imbalance_ratio=1.724518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127718 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=93652 max_load_reduction_pct=0.414323 moved_to_replicas=451238 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[129714, 132383, 127718, 131613, 129998, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127718 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=93652 max_load_reduction_pct=0.414323 moved_to_replicas=451238 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=4202299 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=356864 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=4182511 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376704 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=4218861 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=340864 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=4176327 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=384768 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=4196355 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=360192 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=4205594 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=354304 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=4149171 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=408192 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=4223314 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=336000 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:19:13] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 569.9 | throughput per GPU (TFLOP/s/GPU): 585.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.806283E+00 | loss scale: 1.0 | grad norm: 4.966 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=143616 expected_recv=258776 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=143616 expected_recv=257207 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=143616 expected_recv=260629 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=143616 expected_recv=258776 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=143616 expected_recv=257207 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=143616 expected_recv=261477 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=260608 slack_slots=26624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=259584 slack_slots=27648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=143616 expected_recv=260629 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=143616 expected_recv=261477 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=262912 slack_slots=24320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=143616 expected_recv=254264 expected_overflow=0 exact_recv_per_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=143616 expected_recv=254264 actual_valid_recv=257536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=143616 expected_recv=261141 expected_overflow=0 exact_recv_per_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=257536 slack_slots=29696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=268160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=143616 expected_recv=261141 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=268160 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=143616 expected_recv=257917 expected_overflow=0 exact_recv_per_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=264064 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=143616 expected_recv=257917 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=143616 expected_recv=251155 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=287232 valid_num_tokens=261248 slack_slots=25984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=143616 expected_recv=261044 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=143616 expected_recv=251155 actual_valid_recv=254080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=143616 expected_recv=261044 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=254080 slack_slots=33152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=263808 slack_slots=23424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=143616 expected_recv=261122 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267904 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=143616 expected_recv=261122 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=264448 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=143616 expected_recv=253592 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=143616 expected_recv=253592 actual_valid_recv=256640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264641 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=143616 expected_recv=255091 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=256640 slack_slots=30592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=143616 expected_recv=264641 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=143616 expected_recv=255091 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=287232 valid_num_tokens=257920 slack_slots=29312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=143616 expected_recv=262185 expected_overflow=0 exact_recv_per_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=143616 expected_recv=262185 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=143616 expected_recv=254013 expected_overflow=0 exact_recv_per_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=143616 expected_recv=259986 expected_overflow=0 exact_recv_per_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=265088 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=143616 expected_recv=254013 actual_valid_recv=257024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=143616 expected_recv=261903 expected_overflow=0 exact_recv_per_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=143616 expected_recv=259986 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=257024 slack_slots=30208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=143616 expected_recv=261903 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=264320 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=143616 expected_recv=260314 expected_overflow=0 exact_recv_per_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=143616 expected_recv=256195 expected_overflow=0 exact_recv_per_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=143616 expected_recv=260314 actual_valid_recv=262656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=143616 expected_recv=256195 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=262656 slack_slots=24576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=258688 slack_slots=28544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=143616 expected_recv=256811 expected_overflow=0 exact_recv_per_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=268032 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=143616 expected_recv=256811 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=143616 expected_recv=260680 expected_overflow=0 exact_recv_per_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=259712 slack_slots=27520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=143616 expected_recv=260680 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=143616 expected_recv=257860 expected_overflow=0 exact_recv_per_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=143616 expected_recv=257860 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=143616 expected_recv=260928 expected_overflow=0 exact_recv_per_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=143616 expected_recv=260928 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=143616 expected_recv=258624 expected_overflow=0 exact_recv_per_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=263168 slack_slots=24064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=143616 expected_recv=258624 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=143616 expected_recv=259853 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=261504 slack_slots=25728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=143616 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=143616 expected_recv=259853 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=143616 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=262912 slack_slots=24320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=143616 expected_recv=263538 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=143616 expected_recv=257795 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=143616 expected_recv=263538 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=143616 expected_recv=257795 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=266240 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=143616 expected_recv=262695 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=260736 slack_slots=26496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=143616 expected_recv=262932 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=143616 expected_recv=262695 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=143616 expected_recv=255776 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=143616 expected_recv=262932 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=143616 expected_recv=260806 expected_overflow=0 exact_recv_per_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=264704 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=143616 expected_recv=255776 actual_valid_recv=257664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=143616 expected_recv=261360 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=143616 expected_recv=260806 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=257664 slack_slots=29568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=143616 expected_recv=261360 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=143616 expected_recv=261017 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=143616 expected_recv=259970 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=143616 expected_recv=261017 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=143616 expected_recv=259970 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=263680 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=143616 expected_recv=264729 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=143616 expected_recv=260653 expected_overflow=0 exact_recv_per_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=143616 expected_recv=264729 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=143616 expected_recv=258088 expected_overflow=0 exact_recv_per_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=143616 expected_recv=260653 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=143616 expected_recv=258088 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=143616 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262912 slack_slots=24320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=260480 slack_slots=26752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=143616 expected_recv=264768 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=143616 expected_recv=259881 expected_overflow=0 exact_recv_per_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=143616 expected_recv=259465 expected_overflow=0 exact_recv_per_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=143616 expected_recv=259881 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=143616 expected_recv=259465 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=143616 expected_recv=257500 expected_overflow=0 exact_recv_per_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=261888 slack_slots=25344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=262400 slack_slots=24832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=143616 expected_recv=257500 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=143616 expected_recv=264334 expected_overflow=0 exact_recv_per_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=143616 expected_recv=264334 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=259840 slack_slots=27392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=143616 expected_recv=259477 expected_overflow=0 exact_recv_per_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267776 slack_slots=19456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=143616 expected_recv=256777 expected_overflow=0 exact_recv_per_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=143616 expected_recv=259477 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=143616 expected_recv=256777 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=261632 slack_slots=25600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=287232 valid_num_tokens=259584 slack_slots=27648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267648 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=143616 expected_recv=263309 expected_overflow=0 exact_recv_per_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=143616 expected_recv=257069 expected_overflow=0 exact_recv_per_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=143616 expected_recv=263309 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=265728 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=143616 expected_recv=257069 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=143616 expected_recv=261507 expected_overflow=0 exact_recv_per_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=143616 expected_recv=257096 expected_overflow=0 exact_recv_per_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=143616 expected_recv=260610 expected_overflow=0 exact_recv_per_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=258688 slack_slots=28544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=143616 expected_recv=261507 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=143616 expected_recv=257096 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=143616 expected_recv=258585 expected_overflow=0 exact_recv_per_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=143616 expected_recv=260610 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=259200 slack_slots=28032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=143616 expected_recv=258585 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=287232 valid_num_tokens=262784 slack_slots=24448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=260608 slack_slots=26624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=143616 expected_recv=260441 expected_overflow=0 exact_recv_per_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=143616 expected_recv=260441 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=262528 slack_slots=24704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=143616 expected_recv=257558 expected_overflow=0 exact_recv_per_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266752 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=143616 expected_recv=264770 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=143616 expected_recv=257558 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=266368 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=143616 expected_recv=264770 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=287232 valid_num_tokens=260480 slack_slots=26752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=143616 expected_recv=259200 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267904 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=267264 slack_slots=19968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=143616 expected_recv=259200 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=261376 slack_slots=25856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=143616 expected_recv=260294 expected_overflow=0 exact_recv_per_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=143616 expected_recv=257419 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=143616 expected_recv=256702 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=143616 expected_recv=260294 actual_valid_recv=262656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=266624 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=143616 expected_recv=257419 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=143616 expected_recv=261460 expected_overflow=0 exact_recv_per_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=262656 slack_slots=24576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=143616 expected_recv=256702 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=143616 expected_recv=260844 expected_overflow=0 exact_recv_per_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=259840 slack_slots=27392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=143616 expected_recv=261460 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=287232 valid_num_tokens=259456 slack_slots=27776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=143616 expected_recv=260844 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=143616 expected_recv=258207 expected_overflow=0 exact_recv_per_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=264192 slack_slots=23040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=262272 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=143616 expected_recv=258207 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=143616 expected_recv=262053 expected_overflow=0 exact_recv_per_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=260480 slack_slots=26752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=143616 expected_recv=256143 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=143616 expected_recv=262053 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=263552 slack_slots=23680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=143616 expected_recv=256143 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=143616 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=257920 slack_slots=29312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=143616 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264769 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=287232 valid_num_tokens=267392 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267008 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264763 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267520 slack_slots=19712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267136 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=267904 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[152051, 132906, 122091, 104227, 142797, 112857, 137109, 144538] avg=131072.0000 max=152051 max_rank=0 min=104227 min_rank=3 imbalance_ratio=1.160057 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266496 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=143616 expected_recv=252412 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=143616 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=266880 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=143616 expected_recv=252412 actual_valid_recv=255616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[152051, 132906, 122091, 104227, 142797, 112857, 137109, 144538] avg=131072.0000 max=152051 max_rank=0 min=104227 min_rank=3 imbalance_ratio=1.160057 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=287232 valid_num_tokens=255616 slack_slots=31616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127340 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19667 max_load_reduction_pct=0.129345 moved_to_replicas=95702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[152051, 132906, 122091, 104227, 142797, 112857, 137109, 144538] avg=131072.0000 max=152051 max_rank=0 min=104227 min_rank=3 imbalance_ratio=1.160057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[152051, 132906, 122091, 104227, 142797, 112857, 137109, 144538] avg=131072.0000 max=152051 max_rank=0 min=104227 min_rank=3 imbalance_ratio=1.160057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[152051, 132906, 122091, 104227, 142797, 112857, 137109, 144538] avg=131072.0000 max=152051 max_rank=0 min=104227 min_rank=3 imbalance_ratio=1.160057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127340 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19667 max_load_reduction_pct=0.129345 moved_to_replicas=95702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[152051, 132906, 122091, 104227, 142797, 112857, 137109, 144538] avg=131072.0000 max=152051 max_rank=0 min=104227 min_rank=3 imbalance_ratio=1.160057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127340 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19667 max_load_reduction_pct=0.129345 moved_to_replicas=95702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127340 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19667 max_load_reduction_pct=0.129345 moved_to_replicas=95702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127340 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19667 max_load_reduction_pct=0.129345 moved_to_replicas=95702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[152051, 132906, 122091, 104227, 142797, 112857, 137109, 144538] avg=131072.0000 max=152051 max_rank=0 min=104227 min_rank=3 imbalance_ratio=1.160057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[94288, 96123, 166382, 166009, 82804, 129048, 213524, 100398] avg=131072.0000 max=213524 max_rank=6 min=82804 min_rank=4 imbalance_ratio=1.629059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[152051, 132906, 122091, 104227, 142797, 112857, 137109, 144538] avg=131072.0000 max=152051 max_rank=0 min=104227 min_rank=3 imbalance_ratio=1.160057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127340 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19667 max_load_reduction_pct=0.129345 moved_to_replicas=95702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[94288, 96123, 166382, 166009, 82804, 129048, 213524, 100398] avg=131072.0000 max=213524 max_rank=6 min=82804 min_rank=4 imbalance_ratio=1.629059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=127900 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=81140 max_load_reduction_pct=0.380004 moved_to_replicas=289317 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127340 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19667 max_load_reduction_pct=0.129345 moved_to_replicas=95702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[132384, 132383, 127340, 131075, 132383, 128245, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127340 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=19667 max_load_reduction_pct=0.129345 moved_to_replicas=95702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[94288, 96123, 166382, 166009, 82804, 129048, 213524, 100398] avg=131072.0000 max=213524 max_rank=6 min=82804 min_rank=4 imbalance_ratio=1.629059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[94288, 96123, 166382, 166009, 82804, 129048, 213524, 100398] avg=131072.0000 max=213524 max_rank=6 min=82804 min_rank=4 imbalance_ratio=1.629059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[94288, 96123, 166382, 166009, 82804, 129048, 213524, 100398] avg=131072.0000 max=213524 max_rank=6 min=82804 min_rank=4 imbalance_ratio=1.629059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=127900 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=81140 max_load_reduction_pct=0.380004 moved_to_replicas=289317 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[94288, 96123, 166382, 166009, 82804, 129048, 213524, 100398] avg=131072.0000 max=213524 max_rank=6 min=82804 min_rank=4 imbalance_ratio=1.629059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=127900 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=81140 max_load_reduction_pct=0.380004 moved_to_replicas=289317 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=127900 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=81140 max_load_reduction_pct=0.380004 moved_to_replicas=289317 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=127900 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=81140 max_load_reduction_pct=0.380004 moved_to_replicas=289317 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[109753, 224744, 84978, 208354, 172950, 148286, 49794, 49717] avg=131072.0000 max=224744 max_rank=1 min=49717 min_rank=7 imbalance_ratio=1.714661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[94288, 96123, 166382, 166009, 82804, 129048, 213524, 100398] avg=131072.0000 max=213524 max_rank=6 min=82804 min_rank=4 imbalance_ratio=1.629059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[94288, 96123, 166382, 166009, 82804, 129048, 213524, 100398] avg=131072.0000 max=213524 max_rank=6 min=82804 min_rank=4 imbalance_ratio=1.629059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=127900 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=81140 max_load_reduction_pct=0.380004 moved_to_replicas=289317 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[109753, 224744, 84978, 208354, 172950, 148286, 49794, 49717] avg=131072.0000 max=224744 max_rank=1 min=49717 min_rank=7 imbalance_ratio=1.714661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] avg=131072.0000 max=132384 max_rank=0 min=125536 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=92360 max_load_reduction_pct=0.410956 moved_to_replicas=448433 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=127900 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=81140 max_load_reduction_pct=0.380004 moved_to_replicas=289317 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[127900, 132384, 132383, 132383, 129712, 129048, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=127900 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=81140 max_load_reduction_pct=0.380004 moved_to_replicas=289317 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[109753, 224744, 84978, 208354, 172950, 148286, 49794, 49717] avg=131072.0000 max=224744 max_rank=1 min=49717 min_rank=7 imbalance_ratio=1.714661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[109753, 224744, 84978, 208354, 172950, 148286, 49794, 49717] avg=131072.0000 max=224744 max_rank=1 min=49717 min_rank=7 imbalance_ratio=1.714661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[109753, 224744, 84978, 208354, 172950, 148286, 49794, 49717] avg=131072.0000 max=224744 max_rank=1 min=49717 min_rank=7 imbalance_ratio=1.714661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[109753, 224744, 84978, 208354, 172950, 148286, 49794, 49717] avg=131072.0000 max=224744 max_rank=1 min=49717 min_rank=7 imbalance_ratio=1.714661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] avg=131072.0000 max=132384 max_rank=0 min=125536 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=92360 max_load_reduction_pct=0.410956 moved_to_replicas=448433 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] avg=131072.0000 max=132384 max_rank=0 min=125536 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=92360 max_load_reduction_pct=0.410956 moved_to_replicas=448433 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[144498, 82861, 132610, 195109, 128063, 153886, 152325, 59224] avg=131072.0000 max=195109 max_rank=3 min=59224 min_rank=7 imbalance_ratio=1.488564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] avg=131072.0000 max=132384 max_rank=0 min=125536 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=92360 max_load_reduction_pct=0.410956 moved_to_replicas=448433 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] avg=131072.0000 max=132384 max_rank=0 min=125536 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=92360 max_load_reduction_pct=0.410956 moved_to_replicas=448433 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[109753, 224744, 84978, 208354, 172950, 148286, 49794, 49717] avg=131072.0000 max=224744 max_rank=1 min=49717 min_rank=7 imbalance_ratio=1.714661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[109753, 224744, 84978, 208354, 172950, 148286, 49794, 49717] avg=131072.0000 max=224744 max_rank=1 min=49717 min_rank=7 imbalance_ratio=1.714661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] avg=131072.0000 max=132384 max_rank=0 min=125536 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=92360 max_load_reduction_pct=0.410956 moved_to_replicas=448433 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[144498, 82861, 132610, 195109, 128063, 153886, 152325, 59224] avg=131072.0000 max=195109 max_rank=3 min=59224 min_rank=7 imbalance_ratio=1.488564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] avg=131072.0000 max=132385 max_rank=5 min=126213 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=62724 max_load_reduction_pct=0.321482 moved_to_replicas=234160 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] avg=131072.0000 max=132384 max_rank=0 min=125536 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=92360 max_load_reduction_pct=0.410956 moved_to_replicas=448433 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[132384, 132383, 125536, 132383, 132384, 132383, 128740, 132383] avg=131072.0000 max=132384 max_rank=0 min=125536 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=92360 max_load_reduction_pct=0.410956 moved_to_replicas=448433 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[144498, 82861, 132610, 195109, 128063, 153886, 152325, 59224] avg=131072.0000 max=195109 max_rank=3 min=59224 min_rank=7 imbalance_ratio=1.488564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[144498, 82861, 132610, 195109, 128063, 153886, 152325, 59224] avg=131072.0000 max=195109 max_rank=3 min=59224 min_rank=7 imbalance_ratio=1.488564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[144498, 82861, 132610, 195109, 128063, 153886, 152325, 59224] avg=131072.0000 max=195109 max_rank=3 min=59224 min_rank=7 imbalance_ratio=1.488564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[144498, 82861, 132610, 195109, 128063, 153886, 152325, 59224] avg=131072.0000 max=195109 max_rank=3 min=59224 min_rank=7 imbalance_ratio=1.488564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] avg=131072.0000 max=132385 max_rank=5 min=126213 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=62724 max_load_reduction_pct=0.321482 moved_to_replicas=234160 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[68330, 130906, 193778, 114346, 162052, 98941, 161122, 119101] avg=131072.0000 max=193778 max_rank=2 min=68330 min_rank=0 imbalance_ratio=1.478409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] avg=131072.0000 max=132385 max_rank=5 min=126213 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=62724 max_load_reduction_pct=0.321482 moved_to_replicas=234160 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] avg=131072.0000 max=132385 max_rank=5 min=126213 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=62724 max_load_reduction_pct=0.321482 moved_to_replicas=234160 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] avg=131072.0000 max=132385 max_rank=5 min=126213 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=62724 max_load_reduction_pct=0.321482 moved_to_replicas=234160 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[144498, 82861, 132610, 195109, 128063, 153886, 152325, 59224] avg=131072.0000 max=195109 max_rank=3 min=59224 min_rank=7 imbalance_ratio=1.488564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[144498, 82861, 132610, 195109, 128063, 153886, 152325, 59224] avg=131072.0000 max=195109 max_rank=3 min=59224 min_rank=7 imbalance_ratio=1.488564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] avg=131072.0000 max=132385 max_rank=5 min=126213 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=62724 max_load_reduction_pct=0.321482 moved_to_replicas=234160 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] avg=131072.0000 max=132384 max_rank=2 min=127572 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.316826 moved_to_replicas=240793 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[68330, 130906, 193778, 114346, 162052, 98941, 161122, 119101] avg=131072.0000 max=193778 max_rank=2 min=68330 min_rank=0 imbalance_ratio=1.478409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] avg=131072.0000 max=132385 max_rank=5 min=126213 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=62724 max_load_reduction_pct=0.321482 moved_to_replicas=234160 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 128063, 132385, 132382, 126213] avg=131072.0000 max=132385 max_rank=5 min=126213 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=62724 max_load_reduction_pct=0.321482 moved_to_replicas=234160 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[68330, 130906, 193778, 114346, 162052, 98941, 161122, 119101] avg=131072.0000 max=193778 max_rank=2 min=68330 min_rank=0 imbalance_ratio=1.478409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[68330, 130906, 193778, 114346, 162052, 98941, 161122, 119101] avg=131072.0000 max=193778 max_rank=2 min=68330 min_rank=0 imbalance_ratio=1.478409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[68330, 130906, 193778, 114346, 162052, 98941, 161122, 119101] avg=131072.0000 max=193778 max_rank=2 min=68330 min_rank=0 imbalance_ratio=1.478409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[68330, 130906, 193778, 114346, 162052, 98941, 161122, 119101] avg=131072.0000 max=193778 max_rank=2 min=68330 min_rank=0 imbalance_ratio=1.478409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] avg=131072.0000 max=132384 max_rank=2 min=127572 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.316826 moved_to_replicas=240793 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[94539, 98341, 130255, 158650, 165379, 251079, 107544, 42789] avg=131072.0000 max=251079 max_rank=5 min=42789 min_rank=7 imbalance_ratio=1.915581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] avg=131072.0000 max=132384 max_rank=2 min=127572 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.316826 moved_to_replicas=240793 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] avg=131072.0000 max=132384 max_rank=2 min=127572 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.316826 moved_to_replicas=240793 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] avg=131072.0000 max=132384 max_rank=2 min=127572 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.316826 moved_to_replicas=240793 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] avg=131072.0000 max=132384 max_rank=2 min=127572 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.316826 moved_to_replicas=240793 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[68330, 130906, 193778, 114346, 162052, 98941, 161122, 119101] avg=131072.0000 max=193778 max_rank=2 min=68330 min_rank=0 imbalance_ratio=1.478409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[68330, 130906, 193778, 114346, 162052, 98941, 161122, 119101] avg=131072.0000 max=193778 max_rank=2 min=68330 min_rank=0 imbalance_ratio=1.478409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] avg=131072.0000 max=132384 max_rank=6 min=127302 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=118695 max_load_reduction_pct=0.472740 moved_to_replicas=359314 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[94539, 98341, 130255, 158650, 165379, 251079, 107544, 42789] avg=131072.0000 max=251079 max_rank=5 min=42789 min_rank=7 imbalance_ratio=1.915581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] avg=131072.0000 max=132384 max_rank=2 min=127572 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.316826 moved_to_replicas=240793 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[130762, 130906, 132384, 132382, 132383, 127572, 132383, 129804] avg=131072.0000 max=132384 max_rank=2 min=127572 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.316826 moved_to_replicas=240793 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[94539, 98341, 130255, 158650, 165379, 251079, 107544, 42789] avg=131072.0000 max=251079 max_rank=5 min=42789 min_rank=7 imbalance_ratio=1.915581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[94539, 98341, 130255, 158650, 165379, 251079, 107544, 42789] avg=131072.0000 max=251079 max_rank=5 min=42789 min_rank=7 imbalance_ratio=1.915581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[94539, 98341, 130255, 158650, 165379, 251079, 107544, 42789] avg=131072.0000 max=251079 max_rank=5 min=42789 min_rank=7 imbalance_ratio=1.915581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[94539, 98341, 130255, 158650, 165379, 251079, 107544, 42789] avg=131072.0000 max=251079 max_rank=5 min=42789 min_rank=7 imbalance_ratio=1.915581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] avg=131072.0000 max=132384 max_rank=6 min=127302 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=118695 max_load_reduction_pct=0.472740 moved_to_replicas=359314 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[120755, 81019, 233936, 129881, 175630, 117543, 157007, 32805] avg=131072.0000 max=233936 max_rank=2 min=32805 min_rank=7 imbalance_ratio=1.784790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] avg=131072.0000 max=132384 max_rank=6 min=127302 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=118695 max_load_reduction_pct=0.472740 moved_to_replicas=359314 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] avg=131072.0000 max=132384 max_rank=6 min=127302 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=118695 max_load_reduction_pct=0.472740 moved_to_replicas=359314 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] avg=131072.0000 max=132384 max_rank=6 min=127302 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=118695 max_load_reduction_pct=0.472740 moved_to_replicas=359314 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] avg=131072.0000 max=132384 max_rank=6 min=127302 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=118695 max_load_reduction_pct=0.472740 moved_to_replicas=359314 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[94539, 98341, 130255, 158650, 165379, 251079, 107544, 42789] avg=131072.0000 max=251079 max_rank=5 min=42789 min_rank=7 imbalance_ratio=1.915581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[94539, 98341, 130255, 158650, 165379, 251079, 107544, 42789] avg=131072.0000 max=251079 max_rank=5 min=42789 min_rank=7 imbalance_ratio=1.915581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126241 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=101553 max_load_reduction_pct=0.434106 moved_to_replicas=347388 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[120755, 81019, 233936, 129881, 175630, 117543, 157007, 32805] avg=131072.0000 max=233936 max_rank=2 min=32805 min_rank=7 imbalance_ratio=1.784790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] avg=131072.0000 max=132384 max_rank=6 min=127302 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=118695 max_load_reduction_pct=0.472740 moved_to_replicas=359314 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[127302, 129103, 130255, 132382, 132383, 132383, 132384, 132384] avg=131072.0000 max=132384 max_rank=6 min=127302 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=118695 max_load_reduction_pct=0.472740 moved_to_replicas=359314 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[120755, 81019, 233936, 129881, 175630, 117543, 157007, 32805] avg=131072.0000 max=233936 max_rank=2 min=32805 min_rank=7 imbalance_ratio=1.784790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[120755, 81019, 233936, 129881, 175630, 117543, 157007, 32805] avg=131072.0000 max=233936 max_rank=2 min=32805 min_rank=7 imbalance_ratio=1.784790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126241 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=101553 max_load_reduction_pct=0.434106 moved_to_replicas=347388 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[120755, 81019, 233936, 129881, 175630, 117543, 157007, 32805] avg=131072.0000 max=233936 max_rank=2 min=32805 min_rank=7 imbalance_ratio=1.784790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[120755, 81019, 233936, 129881, 175630, 117543, 157007, 32805] avg=131072.0000 max=233936 max_rank=2 min=32805 min_rank=7 imbalance_ratio=1.784790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[86411, 67962, 208793, 97990, 98367, 128227, 65843, 294983] avg=131072.0000 max=294983 max_rank=7 min=65843 min_rank=6 imbalance_ratio=2.250542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126241 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=101553 max_load_reduction_pct=0.434106 moved_to_replicas=347388 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126241 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=101553 max_load_reduction_pct=0.434106 moved_to_replicas=347388 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[120755, 81019, 233936, 129881, 175630, 117543, 157007, 32805] avg=131072.0000 max=233936 max_rank=2 min=32805 min_rank=7 imbalance_ratio=1.784790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[120755, 81019, 233936, 129881, 175630, 117543, 157007, 32805] avg=131072.0000 max=233936 max_rank=2 min=32805 min_rank=7 imbalance_ratio=1.784790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126241 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=101553 max_load_reduction_pct=0.434106 moved_to_replicas=347388 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126241 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=101553 max_load_reduction_pct=0.434106 moved_to_replicas=347388 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] avg=131072.0000 max=132383 max_rank=1 min=128227 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=162600 max_load_reduction_pct=0.551218 moved_to_replicas=480088 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[86411, 67962, 208793, 97990, 98367, 128227, 65843, 294983] avg=131072.0000 max=294983 max_rank=7 min=65843 min_rank=6 imbalance_ratio=2.250542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[86411, 67962, 208793, 97990, 98367, 128227, 65843, 294983] avg=131072.0000 max=294983 max_rank=7 min=65843 min_rank=6 imbalance_ratio=2.250542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126241 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=101553 max_load_reduction_pct=0.434106 moved_to_replicas=347388 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[130540, 126241, 132383, 129881, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126241 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=101553 max_load_reduction_pct=0.434106 moved_to_replicas=347388 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[86411, 67962, 208793, 97990, 98367, 128227, 65843, 294983] avg=131072.0000 max=294983 max_rank=7 min=65843 min_rank=6 imbalance_ratio=2.250542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] avg=131072.0000 max=132383 max_rank=1 min=128227 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=162600 max_load_reduction_pct=0.551218 moved_to_replicas=480088 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[123792, 142587, 129140, 112345, 110226, 130016, 132346, 168124] avg=131072.0000 max=168124 max_rank=7 min=110226 min_rank=4 imbalance_ratio=1.282684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[86411, 67962, 208793, 97990, 98367, 128227, 65843, 294983] avg=131072.0000 max=294983 max_rank=7 min=65843 min_rank=6 imbalance_ratio=2.250542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[86411, 67962, 208793, 97990, 98367, 128227, 65843, 294983] avg=131072.0000 max=294983 max_rank=7 min=65843 min_rank=6 imbalance_ratio=2.250542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] avg=131072.0000 max=132383 max_rank=1 min=128227 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=162600 max_load_reduction_pct=0.551218 moved_to_replicas=480088 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] avg=131072.0000 max=132383 max_rank=1 min=128227 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=162600 max_load_reduction_pct=0.551218 moved_to_replicas=480088 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[86411, 67962, 208793, 97990, 98367, 128227, 65843, 294983] avg=131072.0000 max=294983 max_rank=7 min=65843 min_rank=6 imbalance_ratio=2.250542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[86411, 67962, 208793, 97990, 98367, 128227, 65843, 294983] avg=131072.0000 max=294983 max_rank=7 min=65843 min_rank=6 imbalance_ratio=2.250542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] avg=131072.0000 max=132383 max_rank=0 min=127542 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35741 max_load_reduction_pct=0.212587 moved_to_replicas=86076 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] avg=131072.0000 max=132383 max_rank=1 min=128227 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=162600 max_load_reduction_pct=0.551218 moved_to_replicas=480088 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] avg=131072.0000 max=132383 max_rank=1 min=128227 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=162600 max_load_reduction_pct=0.551218 moved_to_replicas=480088 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[123792, 142587, 129140, 112345, 110226, 130016, 132346, 168124] avg=131072.0000 max=168124 max_rank=7 min=110226 min_rank=4 imbalance_ratio=1.282684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[123792, 142587, 129140, 112345, 110226, 130016, 132346, 168124] avg=131072.0000 max=168124 max_rank=7 min=110226 min_rank=4 imbalance_ratio=1.282684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] avg=131072.0000 max=132383 max_rank=1 min=128227 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=162600 max_load_reduction_pct=0.551218 moved_to_replicas=480088 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[131156, 132383, 132383, 130745, 129932, 128227, 131367, 132383] avg=131072.0000 max=132383 max_rank=1 min=128227 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=162600 max_load_reduction_pct=0.551218 moved_to_replicas=480088 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[123792, 142587, 129140, 112345, 110226, 130016, 132346, 168124] avg=131072.0000 max=168124 max_rank=7 min=110226 min_rank=4 imbalance_ratio=1.282684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] avg=131072.0000 max=132383 max_rank=0 min=127542 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35741 max_load_reduction_pct=0.212587 moved_to_replicas=86076 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[160774, 96898, 118056, 97614, 18530, 228709, 99126, 228869] avg=131072.0000 max=228869 max_rank=7 min=18530 min_rank=4 imbalance_ratio=1.746132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[123792, 142587, 129140, 112345, 110226, 130016, 132346, 168124] avg=131072.0000 max=168124 max_rank=7 min=110226 min_rank=4 imbalance_ratio=1.282684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] avg=131072.0000 max=132383 max_rank=0 min=127542 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35741 max_load_reduction_pct=0.212587 moved_to_replicas=86076 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[123792, 142587, 129140, 112345, 110226, 130016, 132346, 168124] avg=131072.0000 max=168124 max_rank=7 min=110226 min_rank=4 imbalance_ratio=1.282684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] avg=131072.0000 max=132383 max_rank=0 min=127542 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35741 max_load_reduction_pct=0.212587 moved_to_replicas=86076 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[123792, 142587, 129140, 112345, 110226, 130016, 132346, 168124] avg=131072.0000 max=168124 max_rank=7 min=110226 min_rank=4 imbalance_ratio=1.282684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] avg=131072.0000 max=132385 max_rank=0 min=129079 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96484 max_load_reduction_pct=0.421569 moved_to_replicas=440833 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[123792, 142587, 129140, 112345, 110226, 130016, 132346, 168124] avg=131072.0000 max=168124 max_rank=7 min=110226 min_rank=4 imbalance_ratio=1.282684 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] avg=131072.0000 max=132383 max_rank=0 min=127542 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35741 max_load_reduction_pct=0.212587 moved_to_replicas=86076 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[160774, 96898, 118056, 97614, 18530, 228709, 99126, 228869] avg=131072.0000 max=228869 max_rank=7 min=18530 min_rank=4 imbalance_ratio=1.746132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] avg=131072.0000 max=132383 max_rank=0 min=127542 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35741 max_load_reduction_pct=0.212587 moved_to_replicas=86076 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[160774, 96898, 118056, 97614, 18530, 228709, 99126, 228869] avg=131072.0000 max=228869 max_rank=7 min=18530 min_rank=4 imbalance_ratio=1.746132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] avg=131072.0000 max=132383 max_rank=0 min=127542 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35741 max_load_reduction_pct=0.212587 moved_to_replicas=86076 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[132383, 132383, 129140, 127542, 132383, 130016, 132346, 132383] avg=131072.0000 max=132383 max_rank=0 min=127542 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=35741 max_load_reduction_pct=0.212587 moved_to_replicas=86076 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[160774, 96898, 118056, 97614, 18530, 228709, 99126, 228869] avg=131072.0000 max=228869 max_rank=7 min=18530 min_rank=4 imbalance_ratio=1.746132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] avg=131072.0000 max=132385 max_rank=0 min=129079 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96484 max_load_reduction_pct=0.421569 moved_to_replicas=440833 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[33662, 196590, 63288, 165989, 64370, 197407, 98390, 228880] avg=131072.0000 max=228880 max_rank=7 min=33662 min_rank=0 imbalance_ratio=1.746216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[160774, 96898, 118056, 97614, 18530, 228709, 99126, 228869] avg=131072.0000 max=228869 max_rank=7 min=18530 min_rank=4 imbalance_ratio=1.746132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] avg=131072.0000 max=132385 max_rank=0 min=129079 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96484 max_load_reduction_pct=0.421569 moved_to_replicas=440833 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[160774, 96898, 118056, 97614, 18530, 228709, 99126, 228869] avg=131072.0000 max=228869 max_rank=7 min=18530 min_rank=4 imbalance_ratio=1.746132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] avg=131072.0000 max=132385 max_rank=0 min=129079 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96484 max_load_reduction_pct=0.421569 moved_to_replicas=440833 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[160774, 96898, 118056, 97614, 18530, 228709, 99126, 228869] avg=131072.0000 max=228869 max_rank=7 min=18530 min_rank=4 imbalance_ratio=1.746132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] avg=131072.0000 max=132383 max_rank=1 min=128445 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96497 max_load_reduction_pct=0.421605 moved_to_replicas=519348 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[160774, 96898, 118056, 97614, 18530, 228709, 99126, 228869] avg=131072.0000 max=228869 max_rank=7 min=18530 min_rank=4 imbalance_ratio=1.746132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] avg=131072.0000 max=132385 max_rank=0 min=129079 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96484 max_load_reduction_pct=0.421569 moved_to_replicas=440833 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[33662, 196590, 63288, 165989, 64370, 197407, 98390, 228880] avg=131072.0000 max=228880 max_rank=7 min=33662 min_rank=0 imbalance_ratio=1.746216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] avg=131072.0000 max=132385 max_rank=0 min=129079 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96484 max_load_reduction_pct=0.421569 moved_to_replicas=440833 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[33662, 196590, 63288, 165989, 64370, 197407, 98390, 228880] avg=131072.0000 max=228880 max_rank=7 min=33662 min_rank=0 imbalance_ratio=1.746216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] avg=131072.0000 max=132385 max_rank=0 min=129079 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96484 max_load_reduction_pct=0.421569 moved_to_replicas=440833 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[132385, 129661, 129079, 130374, 132382, 132383, 129929, 132383] avg=131072.0000 max=132385 max_rank=0 min=129079 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96484 max_load_reduction_pct=0.421569 moved_to_replicas=440833 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[33662, 196590, 63288, 165989, 64370, 197407, 98390, 228880] avg=131072.0000 max=228880 max_rank=7 min=33662 min_rank=0 imbalance_ratio=1.746216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[261730, 115910, 108986, 158827, 97762, 142234, 82477, 80650] avg=131072.0000 max=261730 max_rank=0 min=80650 min_rank=7 imbalance_ratio=1.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] avg=131072.0000 max=132383 max_rank=1 min=128445 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96497 max_load_reduction_pct=0.421605 moved_to_replicas=519348 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[33662, 196590, 63288, 165989, 64370, 197407, 98390, 228880] avg=131072.0000 max=228880 max_rank=7 min=33662 min_rank=0 imbalance_ratio=1.746216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] avg=131072.0000 max=132383 max_rank=1 min=128445 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96497 max_load_reduction_pct=0.421605 moved_to_replicas=519348 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[33662, 196590, 63288, 165989, 64370, 197407, 98390, 228880] avg=131072.0000 max=228880 max_rank=7 min=33662 min_rank=0 imbalance_ratio=1.746216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[33662, 196590, 63288, 165989, 64370, 197407, 98390, 228880] avg=131072.0000 max=228880 max_rank=7 min=33662 min_rank=0 imbalance_ratio=1.746216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] avg=131072.0000 max=132383 max_rank=1 min=128445 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96497 max_load_reduction_pct=0.421605 moved_to_replicas=519348 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[33662, 196590, 63288, 165989, 64370, 197407, 98390, 228880] avg=131072.0000 max=228880 max_rank=7 min=33662 min_rank=0 imbalance_ratio=1.746216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] avg=131072.0000 max=132385 max_rank=5 min=125210 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=129345 max_load_reduction_pct=0.494192 moved_to_replicas=328700 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] avg=131072.0000 max=132383 max_rank=1 min=128445 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96497 max_load_reduction_pct=0.421605 moved_to_replicas=519348 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] avg=131072.0000 max=132383 max_rank=1 min=128445 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96497 max_load_reduction_pct=0.421605 moved_to_replicas=519348 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[261730, 115910, 108986, 158827, 97762, 142234, 82477, 80650] avg=131072.0000 max=261730 max_rank=0 min=80650 min_rank=7 imbalance_ratio=1.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[261730, 115910, 108986, 158827, 97762, 142234, 82477, 80650] avg=131072.0000 max=261730 max_rank=0 min=80650 min_rank=7 imbalance_ratio=1.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] avg=131072.0000 max=132383 max_rank=1 min=128445 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96497 max_load_reduction_pct=0.421605 moved_to_replicas=519348 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[131951, 132383, 128814, 132383, 128445, 132383, 129834, 132383] avg=131072.0000 max=132383 max_rank=1 min=128445 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96497 max_load_reduction_pct=0.421605 moved_to_replicas=519348 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[261730, 115910, 108986, 158827, 97762, 142234, 82477, 80650] avg=131072.0000 max=261730 max_rank=0 min=80650 min_rank=7 imbalance_ratio=1.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[123795, 229254, 32902, 130876, 128574, 172906, 164246, 66023] avg=131072.0000 max=229254 max_rank=1 min=32902 min_rank=2 imbalance_ratio=1.749069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[261730, 115910, 108986, 158827, 97762, 142234, 82477, 80650] avg=131072.0000 max=261730 max_rank=0 min=80650 min_rank=7 imbalance_ratio=1.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] avg=131072.0000 max=132385 max_rank=5 min=125210 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=129345 max_load_reduction_pct=0.494192 moved_to_replicas=328700 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] avg=131072.0000 max=132385 max_rank=5 min=125210 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=129345 max_load_reduction_pct=0.494192 moved_to_replicas=328700 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[261730, 115910, 108986, 158827, 97762, 142234, 82477, 80650] avg=131072.0000 max=261730 max_rank=0 min=80650 min_rank=7 imbalance_ratio=1.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[261730, 115910, 108986, 158827, 97762, 142234, 82477, 80650] avg=131072.0000 max=261730 max_rank=0 min=80650 min_rank=7 imbalance_ratio=1.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[261730, 115910, 108986, 158827, 97762, 142234, 82477, 80650] avg=131072.0000 max=261730 max_rank=0 min=80650 min_rank=7 imbalance_ratio=1.996841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] avg=131072.0000 max=132383 max_rank=1 min=128574 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96871 max_load_reduction_pct=0.422549 moved_to_replicas=337881 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] avg=131072.0000 max=132385 max_rank=5 min=125210 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=129345 max_load_reduction_pct=0.494192 moved_to_replicas=328700 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] avg=131072.0000 max=132385 max_rank=5 min=125210 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=129345 max_load_reduction_pct=0.494192 moved_to_replicas=328700 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] avg=131072.0000 max=132385 max_rank=5 min=125210 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=129345 max_load_reduction_pct=0.494192 moved_to_replicas=328700 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[123795, 229254, 32902, 130876, 128574, 172906, 164246, 66023] avg=131072.0000 max=229254 max_rank=1 min=32902 min_rank=2 imbalance_ratio=1.749069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[123795, 229254, 32902, 130876, 128574, 172906, 164246, 66023] avg=131072.0000 max=229254 max_rank=1 min=32902 min_rank=2 imbalance_ratio=1.749069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] avg=131072.0000 max=132385 max_rank=5 min=125210 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=129345 max_load_reduction_pct=0.494192 moved_to_replicas=328700 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[132383, 125210, 132384, 132384, 130521, 132385, 132382, 130927] avg=131072.0000 max=132385 max_rank=5 min=125210 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=129345 max_load_reduction_pct=0.494192 moved_to_replicas=328700 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[67851, 195953, 567, 223709, 87229, 223469, 119841, 129957] avg=131072.0000 max=223709 max_rank=3 min=567 min_rank=2 imbalance_ratio=1.706764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[123795, 229254, 32902, 130876, 128574, 172906, 164246, 66023] avg=131072.0000 max=229254 max_rank=1 min=32902 min_rank=2 imbalance_ratio=1.749069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[123795, 229254, 32902, 130876, 128574, 172906, 164246, 66023] avg=131072.0000 max=229254 max_rank=1 min=32902 min_rank=2 imbalance_ratio=1.749069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] avg=131072.0000 max=132383 max_rank=1 min=128574 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96871 max_load_reduction_pct=0.422549 moved_to_replicas=337881 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] avg=131072.0000 max=132383 max_rank=1 min=128574 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96871 max_load_reduction_pct=0.422549 moved_to_replicas=337881 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[123795, 229254, 32902, 130876, 128574, 172906, 164246, 66023] avg=131072.0000 max=229254 max_rank=1 min=32902 min_rank=2 imbalance_ratio=1.749069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] avg=131072.0000 max=132386 max_rank=2 min=124321 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=91323 max_load_reduction_pct=0.408222 moved_to_replicas=492148 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[123795, 229254, 32902, 130876, 128574, 172906, 164246, 66023] avg=131072.0000 max=229254 max_rank=1 min=32902 min_rank=2 imbalance_ratio=1.749069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[123795, 229254, 32902, 130876, 128574, 172906, 164246, 66023] avg=131072.0000 max=229254 max_rank=1 min=32902 min_rank=2 imbalance_ratio=1.749069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] avg=131072.0000 max=132383 max_rank=1 min=128574 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96871 max_load_reduction_pct=0.422549 moved_to_replicas=337881 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] avg=131072.0000 max=132383 max_rank=1 min=128574 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96871 max_load_reduction_pct=0.422549 moved_to_replicas=337881 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] avg=131072.0000 max=132383 max_rank=1 min=128574 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96871 max_load_reduction_pct=0.422549 moved_to_replicas=337881 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[67851, 195953, 567, 223709, 87229, 223469, 119841, 129957] avg=131072.0000 max=223709 max_rank=3 min=567 min_rank=2 imbalance_ratio=1.706764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] avg=131072.0000 max=132383 max_rank=1 min=128574 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96871 max_load_reduction_pct=0.422549 moved_to_replicas=337881 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[67851, 195953, 567, 223709, 87229, 223469, 119841, 129957] avg=131072.0000 max=223709 max_rank=3 min=567 min_rank=2 imbalance_ratio=1.706764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[129462, 132383, 132381, 130876, 128574, 132383, 132383, 130134] avg=131072.0000 max=132383 max_rank=1 min=128574 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96871 max_load_reduction_pct=0.422549 moved_to_replicas=337881 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[98688, 131045, 72105, 98195, 157216, 153799, 229259, 108269] avg=131072.0000 max=229259 max_rank=6 min=72105 min_rank=2 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[67851, 195953, 567, 223709, 87229, 223469, 119841, 129957] avg=131072.0000 max=223709 max_rank=3 min=567 min_rank=2 imbalance_ratio=1.706764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[67851, 195953, 567, 223709, 87229, 223469, 119841, 129957] avg=131072.0000 max=223709 max_rank=3 min=567 min_rank=2 imbalance_ratio=1.706764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] avg=131072.0000 max=132386 max_rank=2 min=124321 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=91323 max_load_reduction_pct=0.408222 moved_to_replicas=492148 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[67851, 195953, 567, 223709, 87229, 223469, 119841, 129957] avg=131072.0000 max=223709 max_rank=3 min=567 min_rank=2 imbalance_ratio=1.706764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] avg=131072.0000 max=132386 max_rank=2 min=124321 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=91323 max_load_reduction_pct=0.408222 moved_to_replicas=492148 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] avg=131072.0000 max=132383 max_rank=6 min=129686 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=288456 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[67851, 195953, 567, 223709, 87229, 223469, 119841, 129957] avg=131072.0000 max=223709 max_rank=3 min=567 min_rank=2 imbalance_ratio=1.706764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] avg=131072.0000 max=132386 max_rank=2 min=124321 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=91323 max_load_reduction_pct=0.408222 moved_to_replicas=492148 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] avg=131072.0000 max=132386 max_rank=2 min=124321 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=91323 max_load_reduction_pct=0.408222 moved_to_replicas=492148 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[67851, 195953, 567, 223709, 87229, 223469, 119841, 129957] avg=131072.0000 max=223709 max_rank=3 min=567 min_rank=2 imbalance_ratio=1.706764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] avg=131072.0000 max=132386 max_rank=2 min=124321 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=91323 max_load_reduction_pct=0.408222 moved_to_replicas=492148 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[98688, 131045, 72105, 98195, 157216, 153799, 229259, 108269] avg=131072.0000 max=229259 max_rank=6 min=72105 min_rank=2 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] avg=131072.0000 max=132386 max_rank=2 min=124321 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=91323 max_load_reduction_pct=0.408222 moved_to_replicas=492148 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[98688, 131045, 72105, 98195, 157216, 153799, 229259, 108269] avg=131072.0000 max=229259 max_rank=6 min=72105 min_rank=2 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[46011, 178444, 56100, 129000, 67718, 197459, 196537, 177307] avg=131072.0000 max=197459 max_rank=5 min=46011 min_rank=0 imbalance_ratio=1.506493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[132383, 132383, 132386, 132383, 124321, 132380, 132383, 129957] avg=131072.0000 max=132386 max_rank=2 min=124321 min_rank=4 imbalance_ratio=1.010025 max_load_reduction=91323 max_load_reduction_pct=0.408222 moved_to_replicas=492148 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[98688, 131045, 72105, 98195, 157216, 153799, 229259, 108269] avg=131072.0000 max=229259 max_rank=6 min=72105 min_rank=2 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] avg=131072.0000 max=132383 max_rank=6 min=129686 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=288456 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[98688, 131045, 72105, 98195, 157216, 153799, 229259, 108269] avg=131072.0000 max=229259 max_rank=6 min=72105 min_rank=2 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[98688, 131045, 72105, 98195, 157216, 153799, 229259, 108269] avg=131072.0000 max=229259 max_rank=6 min=72105 min_rank=2 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] avg=131072.0000 max=132383 max_rank=6 min=129686 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=288456 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] avg=131072.0000 max=132385 max_rank=2 min=125277 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=65074 max_load_reduction_pct=0.329557 moved_to_replicas=441771 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[98688, 131045, 72105, 98195, 157216, 153799, 229259, 108269] avg=131072.0000 max=229259 max_rank=6 min=72105 min_rank=2 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] avg=131072.0000 max=132383 max_rank=6 min=129686 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=288456 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[98688, 131045, 72105, 98195, 157216, 153799, 229259, 108269] avg=131072.0000 max=229259 max_rank=6 min=72105 min_rank=2 imbalance_ratio=1.749107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] avg=131072.0000 max=132383 max_rank=6 min=129686 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=288456 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] avg=131072.0000 max=132383 max_rank=6 min=129686 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=288456 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[46011, 178444, 56100, 129000, 67718, 197459, 196537, 177307] avg=131072.0000 max=197459 max_rank=5 min=46011 min_rank=0 imbalance_ratio=1.506493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] avg=131072.0000 max=132383 max_rank=6 min=129686 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=288456 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[46011, 178444, 56100, 129000, 67718, 197459, 196537, 177307] avg=131072.0000 max=197459 max_rank=5 min=46011 min_rank=0 imbalance_ratio=1.506493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[130038, 131045, 129702, 130958, 132382, 132382, 132383, 129686] avg=131072.0000 max=132383 max_rank=6 min=129686 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96876 max_load_reduction_pct=0.422561 moved_to_replicas=288456 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[46011, 178444, 56100, 129000, 67718, 197459, 196537, 177307] avg=131072.0000 max=197459 max_rank=5 min=46011 min_rank=0 imbalance_ratio=1.506493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] avg=131072.0000 max=132385 max_rank=2 min=125277 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=65074 max_load_reduction_pct=0.329557 moved_to_replicas=441771 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[46011, 178444, 56100, 129000, 67718, 197459, 196537, 177307] avg=131072.0000 max=197459 max_rank=5 min=46011 min_rank=0 imbalance_ratio=1.506493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] avg=131072.0000 max=132385 max_rank=2 min=125277 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=65074 max_load_reduction_pct=0.329557 moved_to_replicas=441771 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[46011, 178444, 56100, 129000, 67718, 197459, 196537, 177307] avg=131072.0000 max=197459 max_rank=5 min=46011 min_rank=0 imbalance_ratio=1.506493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[46011, 178444, 56100, 129000, 67718, 197459, 196537, 177307] avg=131072.0000 max=197459 max_rank=5 min=46011 min_rank=0 imbalance_ratio=1.506493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] avg=131072.0000 max=132385 max_rank=2 min=125277 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=65074 max_load_reduction_pct=0.329557 moved_to_replicas=441771 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[46011, 178444, 56100, 129000, 67718, 197459, 196537, 177307] avg=131072.0000 max=197459 max_rank=5 min=46011 min_rank=0 imbalance_ratio=1.506493 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] avg=131072.0000 max=132385 max_rank=2 min=125277 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=65074 max_load_reduction_pct=0.329557 moved_to_replicas=441771 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] avg=131072.0000 max=132385 max_rank=2 min=125277 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=65074 max_load_reduction_pct=0.329557 moved_to_replicas=441771 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] avg=131072.0000 max=132385 max_rank=2 min=125277 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=65074 max_load_reduction_pct=0.329557 moved_to_replicas=441771 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[132383, 132384, 132385, 129000, 125277, 132383, 132383, 132381] avg=131072.0000 max=132385 max_rank=2 min=125277 min_rank=4 imbalance_ratio=1.010017 max_load_reduction=65074 max_load_reduction_pct=0.329557 moved_to_replicas=441771 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=4190240 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=368128 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=4202388 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=351744 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=4191556 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=363648 peak_demand=132386 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=4198516 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=361088 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=4202136 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=353664 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=4175999 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=376448 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=4188465 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=364672 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=4205132 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=350848 peak_demand=132384 required_capacity=1.010010 + [2026-08-30 05:19:13] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 574.9 | throughput per GPU (TFLOP/s/GPU): 579.9 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.689411E+00 | loss scale: 1.0 | grad norm: 4.915 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +[after training is done] datetime: 2026-08-30 05:19:13 +[rank0]:[W830 05:19:14.192931492 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 05:19:16.617446916 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 05:19:16.642609863 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 05:19:16.649604659 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 05:19:16.650912235 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 05:19:16.651054344 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 05:19:16.705274398 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 05:19:16.707475307 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:19:18.965000 509099 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:19:18.965000 509099 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 509196 closing signal SIGTERM +W0830 05:19:18.966000 509099 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 509197 closing signal SIGTERM +W0830 05:19:18.966000 509099 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 509198 closing signal SIGTERM +W0830 05:19:18.967000 509099 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 509199 closing signal SIGTERM +W0830 05:19:18.967000 509099 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 509200 closing signal SIGTERM +W0830 05:19:18.968000 509099 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 509201 closing signal SIGTERM +W0830 05:19:18.969000 509099 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 509202 closing signal SIGTERM +W0830 05:19:18.969000 509099 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 509203 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 509099 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..0e5fa1e59e633cfa162524b79b435d11b6c7840f --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,224 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 819 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2177 +1, 2177 +2, 2177 +3, 2177 +4, 2177 +5, 2177 +6, 2177 +7, 2177 +0, 2903 +1, 2903 +2, 2903 +3, 2903 +4, 2903 +5, 2903 +6, 2903 +7, 2903 +0, 42653 +1, 43135 +2, 43145 +3, 43093 +4, 42693 +5, 43137 +6, 43141 +7, 43137 +0, 43573 +1, 44027 +2, 44023 +3, 44033 +4, 43575 +5, 44035 +6, 44049 +7, 44037 +0, 44429 +1, 44877 +2, 44877 +3, 44877 +4, 44429 +5, 44877 +6, 44877 +7, 44877 +0, 45709 +1, 46157 +2, 46157 +3, 46157 +4, 45709 +5, 46157 +6, 46157 +7, 46157 +0, 47555 +1, 48003 +2, 48003 +3, 48003 +4, 47555 +5, 48003 +6, 48003 +7, 48003 +0, 48843 +1, 49291 +2, 49291 +3, 49291 +4, 48843 +5, 49291 +6, 49291 +7, 49291 +0, 82765 +1, 82813 +2, 82813 +3, 82813 +4, 82785 +5, 82813 +6, 82813 +7, 82813 +0, 82765 +1, 82813 +2, 82813 +3, 82813 +4, 82785 +5, 82813 +6, 82813 +7, 82813 +0, 84045 +1, 84093 +2, 84093 +3, 84093 +4, 84065 +5, 84093 +6, 84093 +7, 84093 +0, 85339 +1, 85387 +2, 85387 +3, 84107 +4, 85359 +5, 85387 +6, 85387 +7, 85387 +0, 95747 +1, 95789 +2, 95855 +3, 95827 +4, 95823 +5, 95855 +6, 95827 +7, 95803 +0, 96763 +1, 96823 +2, 96831 +3, 96831 +4, 96829 +5, 96847 +6, 96839 +7, 96901 +0, 113447 +1, 113515 +2, 112215 +3, 113515 +4, 111627 +5, 113515 +6, 112235 +7, 112175 +0, 120387 +1, 120475 +2, 120475 +3, 120475 +4, 120487 +5, 120475 +6, 120475 +7, 120475 +0, 120387 +1, 120475 +2, 120475 +3, 120475 +4, 120487 +5, 120475 +6, 120475 +7, 120475 +0, 120387 +1, 120475 +2, 120475 +3, 120475 +4, 120487 +5, 120475 +6, 120475 +7, 120475 +0, 119433 +1, 119521 +2, 119521 +3, 119521 +4, 119533 +5, 119521 +6, 119521 +7, 119521 +0, 115193 +1, 114893 +2, 115281 +3, 115281 +4, 115293 +5, 115281 +6, 115281 +7, 115281 +0, 3 +1, 4 +2, 68 +3, 113863 +4, 324 +5, 4 +6, 4 +7, 4 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..42128dd4e7827b4eb1dbd299e2f9796ce2158550 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G4 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=896 +num_experts=128 +topk=32 +experts_per_rank=16 +expert_params=11010048 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=131072 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7c92ea21b0f77fdb49b8e347f1390474f0ea3f05 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log @@ -0,0 +1,9351 @@ +[05:16:53] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G4 / ultraep (slots/rank=16 policy=matched) + E=128 K=32 H'=896 m=16 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:16:54.660000 505547 torch/distributed/run.py:766] +W0830 05:16:54.660000 505547 torch/distributed/run.py:766] ***************************************** +W0830 05:16:54.660000 505547 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:16:54.660000 505547 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 896 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 32 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 1324576 is connected to is connected to is connected to is connected to is connected to is connected to is connected to 7777777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7777 +777 + + + + + +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7[Gloo] Rank peer ranks. Expected number of connected peer ranks is : 57 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W830 05:17:12.060337954 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:17:12.060421087 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:17:12.060486085 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:17:12.060529537 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:17:12.062532666 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:17:12.062782407 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:17:12.063058175 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:17:13.236457429 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.411 seconds +time to initialize megatron (seconds): 7.143 +[after megatron is initialized] datetime: 2026-08-30 05:17:18 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6728847360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42479616 elements, 42479616 padded size): + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight +Params for bucket 3 (42475520 elements, 42475520 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_proj.weight +Params for bucket 4 (42475520 elements, 42475520 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 5 (42475520 elements, 42475520 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (42475520 elements, 42475520 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight +Params for bucket 7 (42475520 elements, 42475520 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 8 (42475520 elements, 42475520 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.weight +Params for bucket 9 (42475520 elements, 42475520 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 10 (42475520 elements, 42475520 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight +Params for bucket 11 (42475520 elements, 42475520 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 12 (42475520 elements, 42475520 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight +Params for bucket 13 (42475520 elements, 42475520 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_qkv.weight +Params for bucket 14 (42475520 elements, 42475520 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (42475520 elements, 42475520 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight +Params for bucket 16 (42475520 elements, 42475520 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.pre_mlp_layernorm.weight +Params for bucket 17 (42475520 elements, 42475520 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 42 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 +Params for bucket 43 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 +Params for bucket 46 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 +Params for bucket 47 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 51 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 52 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 55 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 +Params for bucket 56 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 +Params for bucket 59 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 +Params for bucket 60 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 63 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 +Params for bucket 64 (40370176 elements, 40370176 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 +Params for bucket 66 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 67 (29360128 elements, 29360128 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:17:22 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:17:22 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3688.29, 3696.86) + train/valid/test-data-iterators-setup ..........: (74.17, 89.82) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:17:22 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=163840 expected_recv=263153 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=163840 expected_recv=260948 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=163840 expected_recv=257833 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=163840 expected_recv=263153 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=163840 expected_recv=260948 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=163840 expected_recv=257833 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=163840 expected_recv=256154 expected_overflow=0 exact_recv_per_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=163840 expected_recv=256154 actual_valid_recv=258176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=163840 expected_recv=256210 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=258176 slack_slots=69504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=163840 expected_recv=259978 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=163840 expected_recv=256210 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=163840 expected_recv=258578 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=163840 expected_recv=264374 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=163840 expected_recv=259978 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=258304 slack_slots=69376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=163840 expected_recv=263713 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=163840 expected_recv=258578 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=163840 expected_recv=264374 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=163840 expected_recv=263713 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=163840 expected_recv=259444 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=163840 expected_recv=258494 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=163840 expected_recv=259444 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=163840 expected_recv=258494 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=260608 slack_slots=67072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=163840 expected_recv=255383 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=163840 expected_recv=255383 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=163840 expected_recv=262556 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=257920 slack_slots=69760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=163840 expected_recv=262556 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=163840 expected_recv=264381 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=163840 expected_recv=254815 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=163840 expected_recv=256332 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=163840 expected_recv=264381 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=163840 expected_recv=254815 actual_valid_recv=257024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=257024 slack_slots=70656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=163840 expected_recv=256332 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=258688 slack_slots=68992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=163840 expected_recv=256651 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=163840 expected_recv=259490 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=163840 expected_recv=256651 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=259072 slack_slots=68608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=163840 expected_recv=259490 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=163840 expected_recv=257181 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=163840 expected_recv=257181 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=259328 slack_slots=68352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=163840 expected_recv=258404 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=163840 expected_recv=258404 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=163840 expected_recv=255539 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=163840 expected_recv=259378 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=163840 expected_recv=255539 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=163840 expected_recv=259378 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=257408 slack_slots=70272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=163840 expected_recv=260705 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=163840 expected_recv=260705 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=163840 expected_recv=258766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=163840 expected_recv=258766 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=260864 slack_slots=66816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=163840 expected_recv=257276 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=163840 expected_recv=257276 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=259456 slack_slots=68224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=163840 expected_recv=259806 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=163840 expected_recv=261582 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=163840 expected_recv=259806 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=163840 expected_recv=264526 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=163840 expected_recv=261582 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=163840 expected_recv=264526 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=163840 expected_recv=256300 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=163840 expected_recv=263865 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=163840 expected_recv=256300 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=163840 expected_recv=263865 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=259328 slack_slots=68352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265600 slack_slots=62080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=163840 expected_recv=259852 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=163840 expected_recv=259852 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=163840 expected_recv=260819 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262064 expected_overflow=0 exact_recv_per_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=163840 expected_recv=264291 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=163840 expected_recv=260819 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262064 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=163840 expected_recv=258716 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=163840 expected_recv=264291 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=163840 expected_recv=258716 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=163840 expected_recv=258919 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=163840 expected_recv=258919 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=163840 expected_recv=261075 expected_overflow=0 exact_recv_per_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=163840 expected_recv=259642 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=163840 expected_recv=261075 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=163840 expected_recv=259642 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=163840 expected_recv=261345 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=163840 expected_recv=264700 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=163840 expected_recv=261345 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=163840 expected_recv=264700 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=163840 expected_recv=259231 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=163840 expected_recv=259411 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=163840 expected_recv=259231 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=163840 expected_recv=259411 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=163840 expected_recv=259448 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=163840 expected_recv=257185 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=163840 expected_recv=257721 expected_overflow=0 exact_recv_per_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=261760 slack_slots=65920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=163840 expected_recv=259448 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=163840 expected_recv=257185 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=163840 expected_recv=257721 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=259072 slack_slots=68608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=259840 slack_slots=67840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=163840 expected_recv=255859 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=163840 expected_recv=255859 actual_valid_recv=258048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=163840 expected_recv=260120 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=258048 slack_slots=69632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=163840 expected_recv=257342 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262261 expected_overflow=0 exact_recv_per_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=163840 expected_recv=260120 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=163840 expected_recv=257342 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262261 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=259328 slack_slots=68352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=163840 expected_recv=254323 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=163840 expected_recv=254323 actual_valid_recv=256384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=256384 slack_slots=71296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=163840 expected_recv=263260 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=163840 expected_recv=263260 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=265600 slack_slots=62080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=163840 expected_recv=261275 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=163840 expected_recv=259962 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=163840 expected_recv=261275 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=163840 expected_recv=259962 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=163840 expected_recv=260545 expected_overflow=0 exact_recv_per_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=163840 expected_recv=260545 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=163840 expected_recv=255917 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=163840 expected_recv=259345 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=163840 expected_recv=261699 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=163840 expected_recv=255917 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=163840 expected_recv=259345 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=163840 expected_recv=261699 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=258688 slack_slots=68992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=163840 expected_recv=262797 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=163840 expected_recv=262797 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=265472 slack_slots=62208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=163840 expected_recv=259199 expected_overflow=0 exact_recv_per_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=163840 expected_recv=257916 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=163840 expected_recv=259199 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=163840 expected_recv=257916 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=259968 slack_slots=67712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=163840 expected_recv=257732 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=163840 expected_recv=259695 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=163840 expected_recv=257732 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=163840 expected_recv=259695 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=259840 slack_slots=67840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=163840 expected_recv=258028 expected_overflow=0 exact_recv_per_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=163840 expected_recv=263712 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=163840 expected_recv=258028 actual_valid_recv=260224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=163840 expected_recv=259847 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=163840 expected_recv=263712 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=260224 slack_slots=67456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=163840 expected_recv=259847 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=163840 expected_recv=261867 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=163840 expected_recv=261867 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[99759, 130770, 154636, 100388, 147943, 137556, 158912, 118612] avg=131072.0000 max=158912 max_rank=6 min=99759 min_rank=0 imbalance_ratio=1.212402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[127579, 130770, 132383, 128312, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=127579 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=26529 max_load_reduction_pct=0.166941 moved_to_replicas=139873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[170247, 105276, 79826, 103462, 113818, 213042, 166426, 96479] avg=131072.0000 max=213042 max_rank=5 min=79826 min_rank=2 imbalance_ratio=1.625381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[132384, 129861, 129204, 132382, 132383, 132384, 132383, 127595] avg=131072.0000 max=132384 max_rank=0 min=127595 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=80658 max_load_reduction_pct=0.378601 moved_to_replicas=311259 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[194601, 110681, 103418, 153891, 106892, 170224, 78218, 130651] avg=131072.0000 max=194601 max_rank=0 min=78218 min_rank=6 imbalance_ratio=1.484688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132384, 132383, 132383, 123626, 130651] avg=131072.0000 max=132384 max_rank=3 min=123626 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62217 max_load_reduction_pct=0.319716 moved_to_replicas=246433 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[190506, 113898, 58418, 136923, 127718, 148658, 136048, 136407] avg=131072.0000 max=190506 max_rank=0 min=58418 min_rank=2 imbalance_ratio=1.453445 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[132383, 130172, 128769, 132384, 127718, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=3 min=127718 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=58122 max_load_reduction_pct=0.305093 moved_to_replicas=180664 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[145393, 135767, 164298, 165910, 76128, 107813, 136102, 117165] avg=131072.0000 max=165910 max_rank=3 min=76128 min_rank=4 imbalance_ratio=1.265793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130010, 132383, 132383, 124268] avg=131072.0000 max=132384 max_rank=3 min=124268 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33526 max_load_reduction_pct=0.202073 moved_to_replicas=191850 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[110418, 161109, 162145, 127979, 112183, 156177, 117930, 100635] avg=131072.0000 max=162145 max_rank=2 min=100635 min_rank=7 imbalance_ratio=1.237068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[132384, 132383, 132383, 127979, 128284, 132383, 132383, 130397] avg=131072.0000 max=132384 max_rank=0 min=127979 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29761 max_load_reduction_pct=0.183546 moved_to_replicas=168608 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[104805, 120674, 125766, 159783, 104141, 117852, 145261, 170294] avg=131072.0000 max=170294 max_rank=7 min=104141 min_rank=4 imbalance_ratio=1.299240 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[132383, 132383, 126935, 132382, 131405, 128322, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126935 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=37911 max_load_reduction_pct=0.222621 moved_to_replicas=152584 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[141443, 119838, 178582, 47185, 139968, 158017, 132062, 131481] avg=131072.0000 max=178582 max_rank=2 min=47185 min_rank=3 imbalance_ratio=1.362473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[132383, 127423, 132384, 128077, 132383, 132383, 132062, 131481] avg=131072.0000 max=132384 max_rank=2 min=127423 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=46198 max_load_reduction_pct=0.258693 moved_to_replicas=203686 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[114552, 146679, 88272, 114314, 98677, 127470, 126132, 232480] avg=131072.0000 max=232480 max_rank=7 min=88272 min_rank=2 imbalance_ratio=1.773682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[128691, 132383, 132383, 132383, 130500, 127470, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=127470 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=100096 max_load_reduction_pct=0.430557 moved_to_replicas=247011 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[90508, 141241, 126490, 105388, 139832, 125704, 82657, 236756] avg=131072.0000 max=236756 max_rank=7 min=82657 min_rank=6 imbalance_ratio=1.806305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[128456, 132383, 127260, 132384, 132383, 132383, 130946, 132381] avg=131072.0000 max=132384 max_rank=3 min=127260 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=104372 max_load_reduction_pct=0.440842 moved_to_replicas=206180 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[129878, 76143, 132930, 162067, 156887, 122873, 134430, 133368] avg=131072.0000 max=162067 max_rank=3 min=76143 min_rank=1 imbalance_ratio=1.236473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[129878, 129936, 132383, 132383, 132383, 126847, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126847 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=29684 max_load_reduction_pct=0.183159 moved_to_replicas=152073 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[135700, 64487, 98822, 156435, 145981, 75328, 187458, 184365] avg=131072.0000 max=187458 max_rank=6 min=64487 min_rank=1 imbalance_ratio=1.430191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[132383, 132382, 126199, 132383, 132384, 128079, 132383, 132383] avg=131072.0000 max=132384 max_rank=4 min=126199 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55074 max_load_reduction_pct=0.293794 moved_to_replicas=224811 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[118652, 91976, 137780, 112993, 152692, 114459, 127579, 192445] avg=131072.0000 max=192445 max_rank=7 min=91976 min_rank=1 imbalance_ratio=1.468239 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[128161, 132383, 132382, 132383, 132383, 130921, 127579, 132384] avg=131072.0000 max=132384 max_rank=7 min=127579 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60061 max_load_reduction_pct=0.312094 moved_to_replicas=188868 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[78395, 115693, 190378, 141070, 120628, 165401, 144419, 92592] avg=131072.0000 max=190378 max_rank=2 min=78395 min_rank=0 imbalance_ratio=1.452469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[132384, 128892, 132383, 132382, 129316, 132383, 132382, 128454] avg=131072.0000 max=132384 max_rank=0 min=128454 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57994 max_load_reduction_pct=0.304626 moved_to_replicas=198697 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[125470, 130414, 166087, 120396, 144747, 71601, 154041, 135820] avg=131072.0000 max=166087 max_rank=2 min=71601 min_rank=5 imbalance_ratio=1.267143 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[129284, 130414, 132383, 132384, 132382, 126962, 132384, 132383] avg=131072.0000 max=132384 max_rank=3 min=126962 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33703 max_load_reduction_pct=0.202924 moved_to_replicas=160851 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[153622, 127053, 132132, 122933, 82901, 153510, 144032, 132393] avg=131072.0000 max=153622 max_rank=0 min=82901 min_rank=4 imbalance_ratio=1.172043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[132384, 127063, 132132, 132383, 127465, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=127063 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21238 max_load_reduction_pct=0.138248 moved_to_replicas=104493 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=4196979 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1011200 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=4182172 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1025152 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=4191694 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015424 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=4196424 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1008896 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=4195375 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1010944 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=4194937 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1012864 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=4197711 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1008768 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=4199140 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1007104 peak_demand=132384 required_capacity=1.010010 +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB + [2026-08-30 05:17:51] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 29161.6 | throughput per GPU (TFLOP/s/GPU): 11.4 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102500E+01 | loss scale: 1.0 | grad norm: 44.121 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 133888 | min safe capacity: 1.0215 | +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58074.8916015625 | max allocated: 86257.341796875 | reserved: 87986.0 | max reserved: 87986.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262310 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=163840 expected_recv=255075 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262934 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=163840 expected_recv=257768 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262310 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=163840 expected_recv=255075 actual_valid_recv=257024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262934 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=257024 slack_slots=70656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=163840 expected_recv=257768 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=260352 slack_slots=67328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=163840 expected_recv=263015 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=163840 expected_recv=263015 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=163840 expected_recv=261338 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=163840 expected_recv=264197 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=163840 expected_recv=261338 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=163840 expected_recv=264197 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=163840 expected_recv=253504 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=163840 expected_recv=260799 expected_overflow=0 exact_recv_per_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=163840 expected_recv=253504 actual_valid_recv=255360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=163840 expected_recv=260799 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=255360 slack_slots=72320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=163840 expected_recv=257763 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=163840 expected_recv=257763 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=163840 expected_recv=259677 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=259840 slack_slots=67840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=163840 expected_recv=259677 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=163840 expected_recv=255884 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=163840 expected_recv=255884 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=258304 slack_slots=69376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=163840 expected_recv=261884 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=163840 expected_recv=259174 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=163840 expected_recv=261884 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=163840 expected_recv=255090 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=163840 expected_recv=259174 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=163840 expected_recv=261939 expected_overflow=0 exact_recv_per_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=163840 expected_recv=255090 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=163840 expected_recv=261939 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=257408 slack_slots=70272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=163840 expected_recv=258873 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=163840 expected_recv=258873 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=163840 expected_recv=263554 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=163840 expected_recv=259455 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=163840 expected_recv=256207 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=163840 expected_recv=263554 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=163840 expected_recv=259455 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=163840 expected_recv=256207 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=163840 expected_recv=264084 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=258688 slack_slots=68992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=163840 expected_recv=261642 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=163840 expected_recv=264084 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=163840 expected_recv=261642 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=163840 expected_recv=253734 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=163840 expected_recv=258628 expected_overflow=0 exact_recv_per_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=163840 expected_recv=253734 actual_valid_recv=255872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=163840 expected_recv=258628 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=255872 slack_slots=71808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=163840 expected_recv=261556 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=260480 slack_slots=67200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=163840 expected_recv=257022 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=163840 expected_recv=261556 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=163840 expected_recv=260036 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=163840 expected_recv=257022 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=163840 expected_recv=260036 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=163840 expected_recv=259476 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=259200 slack_slots=68480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=163840 expected_recv=259476 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262795 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=163840 expected_recv=261716 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=163840 expected_recv=259913 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262795 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=163840 expected_recv=261716 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=163840 expected_recv=259418 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=163840 expected_recv=259913 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=163840 expected_recv=259012 expected_overflow=0 exact_recv_per_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=163840 expected_recv=268616 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=163840 expected_recv=259418 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=163840 expected_recv=259012 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=163840 expected_recv=252228 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=163840 expected_recv=269084 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=163840 expected_recv=268616 actual_valid_recv=270720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=163840 expected_recv=252228 actual_valid_recv=254720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=163840 expected_recv=269084 actual_valid_recv=270976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=270720 slack_slots=56960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=163840 expected_recv=254159 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=254720 slack_slots=72960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=270976 slack_slots=56704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=163840 expected_recv=254159 actual_valid_recv=256000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=163840 expected_recv=269083 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=256000 slack_slots=71680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=163840 expected_recv=269080 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=163840 expected_recv=269083 actual_valid_recv=271104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=163840 expected_recv=245820 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=163840 expected_recv=269082 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=271104 slack_slots=56576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=163840 expected_recv=269080 actual_valid_recv=271744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=163840 expected_recv=245820 actual_valid_recv=248320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=163840 expected_recv=269082 actual_valid_recv=271488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=163840 expected_recv=260243 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=271744 slack_slots=55936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=248320 slack_slots=79360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=271488 slack_slots=56192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=163840 expected_recv=258786 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=163840 expected_recv=260243 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=163840 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=163840 expected_recv=258786 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=163840 expected_recv=264763 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=163840 expected_recv=261218 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=163840 expected_recv=261218 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=163840 expected_recv=259975 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262632 expected_overflow=0 exact_recv_per_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=163840 expected_recv=259975 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262632 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=163840 expected_recv=258702 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=163840 expected_recv=259093 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=163840 expected_recv=258702 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=261120 slack_slots=66560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=163840 expected_recv=259093 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=163840 expected_recv=255526 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=163840 expected_recv=255526 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=163840 expected_recv=261932 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=163840 expected_recv=263006 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=257792 slack_slots=69888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=163840 expected_recv=261932 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=163840 expected_recv=263006 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=163840 expected_recv=260510 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=163840 expected_recv=260510 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=163840 expected_recv=252641 expected_overflow=0 exact_recv_per_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=163840 expected_recv=259513 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=163840 expected_recv=254755 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=163840 expected_recv=252641 actual_valid_recv=254848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=163840 expected_recv=259513 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=254848 slack_slots=72832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=163840 expected_recv=254755 actual_valid_recv=256896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=163840 expected_recv=262589 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=163840 expected_recv=261230 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=261760 slack_slots=65920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=256896 slack_slots=70784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=163840 expected_recv=262589 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=163840 expected_recv=261230 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=163840 expected_recv=254964 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=163840 expected_recv=254964 actual_valid_recv=256640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=256640 slack_slots=71040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=163840 expected_recv=255856 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=163840 expected_recv=255856 actual_valid_recv=258176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262674 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=258176 slack_slots=69504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=163840 expected_recv=264593 expected_overflow=0 exact_recv_per_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262674 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=163840 expected_recv=258632 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=163840 expected_recv=259790 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=163840 expected_recv=264593 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=163840 expected_recv=258632 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=163840 expected_recv=259790 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=163840 expected_recv=261659 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=260736 slack_slots=66944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=163840 expected_recv=261659 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=163840 expected_recv=259811 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=163840 expected_recv=259811 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262960 expected_overflow=0 exact_recv_per_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262574 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262960 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=163840 expected_recv=258943 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=163840 expected_recv=260001 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262574 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=265472 slack_slots=62208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=163840 expected_recv=258943 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=163840 expected_recv=260001 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=163840 expected_recv=264352 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=261760 slack_slots=65920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=163840 expected_recv=264352 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=163840 expected_recv=261830 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=163840 expected_recv=259920 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=163840 expected_recv=261830 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=163840 expected_recv=259920 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[99609, 138217, 159386, 101911, 139155, 140334, 152047, 117917] avg=131072.0000 max=159386 max_rank=2 min=99609 min_rank=0 imbalance_ratio=1.216019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[129927, 132383, 132383, 130865, 132384, 132382, 132383, 125869] avg=131072.0000 max=132384 max_rank=4 min=125869 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=27002 max_load_reduction_pct=0.169413 moved_to_replicas=140153 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[179289, 110987, 78675, 104454, 127642, 196927, 153208, 97394] avg=131072.0000 max=196927 max_rank=5 min=78675 min_rank=2 imbalance_ratio=1.502434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[132383, 131811, 130633, 132383, 127642, 132383, 132384, 128957] avg=131072.0000 max=132384 max_rank=6 min=127642 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64543 max_load_reduction_pct=0.327751 moved_to_replicas=275737 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[178236, 103094, 125380, 169211, 114368, 146317, 75219, 136751] avg=131072.0000 max=178236 max_rank=0 min=75219 min_rank=6 imbalance_ratio=1.359833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[132383, 128900, 125380, 132382, 132383, 132383, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=125380 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=45853 max_load_reduction_pct=0.257260 moved_to_replicas=222835 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[223020, 131790, 79145, 137042, 77557, 152667, 122458, 124897] avg=131072.0000 max=223020 max_rank=0 min=77557 min_rank=4 imbalance_ratio=1.701508 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[132383, 131790, 132382, 132383, 132384, 132383, 125315, 129556] avg=131072.0000 max=132384 max_rank=4 min=125315 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=90636 max_load_reduction_pct=0.406403 moved_to_replicas=245121 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[151360, 168225, 172267, 131171, 95636, 102169, 106218, 121530] avg=131072.0000 max=172267 max_rank=2 min=95636 min_rank=4 imbalance_ratio=1.314293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[132383, 132383, 132382, 131171, 127072, 132383, 132383, 128419] avg=131072.0000 max=132383 max_rank=0 min=127072 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39884 max_load_reduction_pct=0.231524 moved_to_replicas=194933 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[109952, 162575, 172450, 114859, 136673, 131701, 113601, 106765] avg=131072.0000 max=172450 max_rank=2 min=106765 min_rank=7 imbalance_ratio=1.315689 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[129258, 132383, 132383, 125702, 132383, 131701, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=125702 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=40067 max_load_reduction_pct=0.232340 moved_to_replicas=172482 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[126944, 121005, 109243, 177399, 86372, 129503, 147815, 150295] avg=131072.0000 max=177399 max_rank=3 min=86372 min_rank=4 imbalance_ratio=1.353447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[126944, 130214, 132384, 132381, 132384, 129503, 132384, 132382] avg=131072.0000 max=132384 max_rank=2 min=126944 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=45015 max_load_reduction_pct=0.253750 moved_to_replicas=141069 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[159150, 122354, 205827, 33711, 141944, 139362, 118697, 127531] avg=131072.0000 max=205827 max_rank=2 min=33711 min_rank=3 imbalance_ratio=1.570335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[132383, 129333, 132383, 131768, 132383, 132383, 130412, 127531] avg=131072.0000 max=132383 max_rank=0 min=127531 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=73444 max_load_reduction_pct=0.356824 moved_to_replicas=241309 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[136645, 153037, 78909, 92550, 76548, 152420, 119815, 238652] avg=131072.0000 max=238652 max_rank=7 min=76548 min_rank=4 imbalance_ratio=1.820770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[132384, 132383, 131088, 128909, 126664, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=0 min=126664 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=106268 max_load_reduction_pct=0.445284 moved_to_replicas=261981 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[111477, 123979, 138763, 90420, 127398, 147527, 98385, 210627] avg=131072.0000 max=210627 max_rank=7 min=90420 min_rank=3 imbalance_ratio=1.606956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[130234, 128835, 132383, 132382, 127592, 132384, 132383, 132383] avg=131072.0000 max=132384 max_rank=5 min=127592 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=78243 max_load_reduction_pct=0.371477 moved_to_replicas=218598 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[118636, 82751, 112075, 151899, 168218, 133422, 154009, 127566] avg=131072.0000 max=168218 max_rank=4 min=82751 min_rank=1 imbalance_ratio=1.283401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[132383, 126711, 132383, 132383, 132384, 132383, 132383, 127566] avg=131072.0000 max=132384 max_rank=4 min=126711 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35834 max_load_reduction_pct=0.213021 moved_to_replicas=166123 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[132299, 73105, 137615, 171615, 124653, 118413, 132544, 158332] avg=131072.0000 max=171615 max_rank=3 min=73105 min_rank=1 imbalance_ratio=1.309319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[132299, 132383, 132383, 132383, 124814, 129548, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124814 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=39232 max_load_reduction_pct=0.228605 moved_to_replicas=140840 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[95128, 137793, 124371, 78856, 154833, 105959, 168833, 182803] avg=131072.0000 max=182803 max_rank=7 min=78856 min_rank=3 imbalance_ratio=1.394676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[124498, 132385, 129779, 132384, 132382, 132383, 132383, 132382] avg=131072.0000 max=132385 max_rank=1 min=124498 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=50418 max_load_reduction_pct=0.275805 moved_to_replicas=182840 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[85790, 117118, 168554, 141015, 132209, 169437, 158051, 76402] avg=131072.0000 max=169437 max_rank=5 min=76402 min_rank=7 imbalance_ratio=1.292702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[130978, 123472, 132382, 132385, 132209, 132384, 132383, 132383] avg=131072.0000 max=132385 max_rank=3 min=123472 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=37052 max_load_reduction_pct=0.218677 moved_to_replicas=198649 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 150311, 120671, 124314, 79313, 160368, 157516] avg=131072.0000 max=160368 max_rank=6 min=79313 min_rank=5 imbalance_ratio=1.223511 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[128654, 127429, 132384, 132383, 130577, 132384, 132382, 132383] avg=131072.0000 max=132384 max_rank=2 min=127429 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=27984 max_load_reduction_pct=0.174499 moved_to_replicas=149564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[135461, 137232, 111961, 121340, 138661, 146940, 118292, 138689] avg=131072.0000 max=146940 max_rank=5 min=111961 min_rank=2 imbalance_ratio=1.121063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[132383, 132384, 129596, 127618, 132383, 132383, 129446, 132383] avg=131072.0000 max=132384 max_rank=1 min=127618 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14556 max_load_reduction_pct=0.099061 moved_to_replicas=83453 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=4185948 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1021440 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=4190415 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016704 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=4217258 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=990720 peak_demand=136233 required_capacity=1.039375 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=4192830 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1013632 peak_demand=136700 required_capacity=1.042938 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=4191399 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014272 peak_demand=136700 required_capacity=1.042938 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=4162769 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1043584 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=4199877 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1006080 peak_demand=136698 required_capacity=1.042923 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=4213936 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=993664 peak_demand=136699 required_capacity=1.042931 + [2026-08-30 05:17:52] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 766.4 | throughput per GPU (TFLOP/s/GPU): 435.0 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101850E+01 | loss scale: 1.0 | grad norm: 45.886 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=163840 expected_recv=261494 expected_overflow=0 exact_recv_per_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=163840 expected_recv=254928 expected_overflow=0 exact_recv_per_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262616 expected_overflow=0 exact_recv_per_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=163840 expected_recv=259053 expected_overflow=0 exact_recv_per_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=163840 expected_recv=261494 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=163840 expected_recv=254928 actual_valid_recv=257152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262616 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=163840 expected_recv=259053 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=257152 slack_slots=70528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=163840 expected_recv=260979 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=163840 expected_recv=260979 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=163840 expected_recv=255038 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=163840 expected_recv=255038 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=163840 expected_recv=257306 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=257408 slack_slots=70272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=163840 expected_recv=257306 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264675 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=163840 expected_recv=256146 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264675 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=163840 expected_recv=256146 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=258688 slack_slots=68992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=163840 expected_recv=252501 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=163840 expected_recv=252501 actual_valid_recv=255232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=163840 expected_recv=257082 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=163840 expected_recv=263180 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=255232 slack_slots=72448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=163840 expected_recv=257082 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=163840 expected_recv=263180 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=259712 slack_slots=67968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=265472 slack_slots=62208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=163840 expected_recv=253059 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=163840 expected_recv=253059 actual_valid_recv=255616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=255616 slack_slots=72064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=163840 expected_recv=259949 expected_overflow=0 exact_recv_per_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=163840 expected_recv=259949 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=163840 expected_recv=259736 expected_overflow=0 exact_recv_per_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=163840 expected_recv=259736 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=163840 expected_recv=259310 expected_overflow=0 exact_recv_per_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=163840 expected_recv=259095 expected_overflow=0 exact_recv_per_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=163840 expected_recv=259310 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=163840 expected_recv=259095 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=163840 expected_recv=255107 expected_overflow=0 exact_recv_per_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=163840 expected_recv=255107 actual_valid_recv=257664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=257664 slack_slots=70016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=163840 expected_recv=260444 expected_overflow=0 exact_recv_per_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=163840 expected_recv=257772 expected_overflow=0 exact_recv_per_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=163840 expected_recv=260444 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=163840 expected_recv=257772 actual_valid_recv=260224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=260224 slack_slots=67456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=163840 expected_recv=259161 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=163840 expected_recv=259161 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=163840 expected_recv=258588 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=163840 expected_recv=256850 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=163840 expected_recv=258588 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=163840 expected_recv=263489 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=163840 expected_recv=259901 expected_overflow=0 exact_recv_per_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=163840 expected_recv=256850 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=268032 slack_slots=59648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=163840 expected_recv=255262 expected_overflow=0 exact_recv_per_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=163840 expected_recv=263489 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=163840 expected_recv=259901 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=259328 slack_slots=68352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=163840 expected_recv=255262 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=163840 expected_recv=261904 expected_overflow=0 exact_recv_per_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=257792 slack_slots=69888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=163840 expected_recv=261904 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=163840 expected_recv=264151 expected_overflow=0 exact_recv_per_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=163840 expected_recv=264151 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=163840 expected_recv=255040 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=163840 expected_recv=261637 expected_overflow=0 exact_recv_per_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=163840 expected_recv=255040 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=163840 expected_recv=256990 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=163840 expected_recv=261637 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=257920 slack_slots=69760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=163840 expected_recv=256990 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=163840 expected_recv=261292 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=259456 slack_slots=68224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=163840 expected_recv=261292 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=163840 expected_recv=259625 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=163840 expected_recv=259625 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=163840 expected_recv=258708 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262159 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=163840 expected_recv=258708 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262159 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=163840 expected_recv=257594 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=163840 expected_recv=258685 expected_overflow=0 exact_recv_per_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=163840 expected_recv=257594 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=163840 expected_recv=258685 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262103 expected_overflow=0 exact_recv_per_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=261120 slack_slots=66560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262103 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262512 expected_overflow=0 exact_recv_per_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262512 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=163840 expected_recv=259326 expected_overflow=0 exact_recv_per_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=163840 expected_recv=263515 expected_overflow=0 exact_recv_per_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=163840 expected_recv=259326 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=163840 expected_recv=264291 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=163840 expected_recv=263515 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=163840 expected_recv=257629 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=163840 expected_recv=264291 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=163840 expected_recv=257629 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=163840 expected_recv=261480 expected_overflow=0 exact_recv_per_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=259968 slack_slots=67712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=163840 expected_recv=261480 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=163840 expected_recv=258606 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=163840 expected_recv=258606 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=163840 expected_recv=257558 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=163840 expected_recv=257558 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=163840 expected_recv=256970 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=163840 expected_recv=261387 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=163840 expected_recv=256970 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=163840 expected_recv=261387 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=259072 slack_slots=68608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=163840 expected_recv=256593 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=163840 expected_recv=256593 actual_valid_recv=258944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=258944 slack_slots=68736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=163840 expected_recv=263135 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=163840 expected_recv=256052 expected_overflow=0 exact_recv_per_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=163840 expected_recv=263135 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=163840 expected_recv=256052 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=258688 slack_slots=68992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=163840 expected_recv=256025 expected_overflow=0 exact_recv_per_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=163840 expected_recv=256025 actual_valid_recv=258816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=163840 expected_recv=252445 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=258816 slack_slots=68864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=163840 expected_recv=252445 actual_valid_recv=255360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=255360 slack_slots=72320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=163840 expected_recv=261248 expected_overflow=0 exact_recv_per_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=163840 expected_recv=261248 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=163840 expected_recv=256112 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=163840 expected_recv=256112 actual_valid_recv=258560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=163840 expected_recv=254125 expected_overflow=0 exact_recv_per_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262287 expected_overflow=0 exact_recv_per_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=258560 slack_slots=69120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=163840 expected_recv=254125 actual_valid_recv=256512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262287 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=256512 slack_slots=71168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=163840 expected_recv=260207 expected_overflow=0 exact_recv_per_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=163840 expected_recv=260207 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=163840 expected_recv=261473 expected_overflow=0 exact_recv_per_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[95466, 133128, 140306, 102769, 151691, 141980, 165622, 117614] avg=131072.0000 max=165622 max_rank=6 min=95466 min_rank=0 imbalance_ratio=1.263596 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=163840 expected_recv=261473 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[95466, 133128, 140306, 102769, 151691, 141980, 165622, 117614] avg=131072.0000 max=165622 max_rank=6 min=95466 min_rank=0 imbalance_ratio=1.263596 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[95466, 133128, 140306, 102769, 151691, 141980, 165622, 117614] avg=131072.0000 max=165622 max_rank=6 min=95466 min_rank=0 imbalance_ratio=1.263596 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] avg=131072.0000 max=132384 max_rank=3 min=126827 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33238 max_load_reduction_pct=0.200686 moved_to_replicas=142246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[95466, 133128, 140306, 102769, 151691, 141980, 165622, 117614] avg=131072.0000 max=165622 max_rank=6 min=95466 min_rank=0 imbalance_ratio=1.263596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] avg=131072.0000 max=132384 max_rank=3 min=126827 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33238 max_load_reduction_pct=0.200686 moved_to_replicas=142246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] avg=131072.0000 max=132384 max_rank=3 min=126827 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33238 max_load_reduction_pct=0.200686 moved_to_replicas=142246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[95466, 133128, 140306, 102769, 151691, 141980, 165622, 117614] avg=131072.0000 max=165622 max_rank=6 min=95466 min_rank=0 imbalance_ratio=1.263596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[95466, 133128, 140306, 102769, 151691, 141980, 165622, 117614] avg=131072.0000 max=165622 max_rank=6 min=95466 min_rank=0 imbalance_ratio=1.263596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] avg=131072.0000 max=132384 max_rank=3 min=126827 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33238 max_load_reduction_pct=0.200686 moved_to_replicas=142246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[95466, 133128, 140306, 102769, 151691, 141980, 165622, 117614] avg=131072.0000 max=165622 max_rank=6 min=95466 min_rank=0 imbalance_ratio=1.263596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[142761, 102162, 66334, 151030, 123653, 210377, 100046, 152213] avg=131072.0000 max=210377 max_rank=5 min=66334 min_rank=2 imbalance_ratio=1.605049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[142761, 102162, 66334, 151030, 123653, 210377, 100046, 152213] avg=131072.0000 max=210377 max_rank=5 min=66334 min_rank=2 imbalance_ratio=1.605049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[95466, 133128, 140306, 102769, 151691, 141980, 165622, 117614] avg=131072.0000 max=165622 max_rank=6 min=95466 min_rank=0 imbalance_ratio=1.263596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] avg=131072.0000 max=132384 max_rank=3 min=126827 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33238 max_load_reduction_pct=0.200686 moved_to_replicas=142246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[142761, 102162, 66334, 151030, 123653, 210377, 100046, 152213] avg=131072.0000 max=210377 max_rank=5 min=66334 min_rank=2 imbalance_ratio=1.605049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] avg=131072.0000 max=132384 max_rank=3 min=126827 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33238 max_load_reduction_pct=0.200686 moved_to_replicas=142246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] avg=131072.0000 max=132384 max_rank=0 min=123844 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=77993 max_load_reduction_pct=0.370730 moved_to_replicas=231133 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] avg=131072.0000 max=132384 max_rank=3 min=126827 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33238 max_load_reduction_pct=0.200686 moved_to_replicas=142246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] avg=131072.0000 max=132384 max_rank=0 min=123844 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=77993 max_load_reduction_pct=0.370730 moved_to_replicas=231133 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[142761, 102162, 66334, 151030, 123653, 210377, 100046, 152213] avg=131072.0000 max=210377 max_rank=5 min=66334 min_rank=2 imbalance_ratio=1.605049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] avg=131072.0000 max=132384 max_rank=0 min=123844 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=77993 max_load_reduction_pct=0.370730 moved_to_replicas=231133 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[127452, 132383, 132383, 132384, 132382, 132382, 132383, 126827] avg=131072.0000 max=132384 max_rank=3 min=126827 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=33238 max_load_reduction_pct=0.200686 moved_to_replicas=142246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[142761, 102162, 66334, 151030, 123653, 210377, 100046, 152213] avg=131072.0000 max=210377 max_rank=5 min=66334 min_rank=2 imbalance_ratio=1.605049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] avg=131072.0000 max=132384 max_rank=0 min=123844 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=77993 max_load_reduction_pct=0.370730 moved_to_replicas=231133 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[142761, 102162, 66334, 151030, 123653, 210377, 100046, 152213] avg=131072.0000 max=210377 max_rank=5 min=66334 min_rank=2 imbalance_ratio=1.605049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[155391, 178330, 118221, 111671, 148152, 126147, 119127, 91537] avg=131072.0000 max=178330 max_rank=1 min=91537 min_rank=7 imbalance_ratio=1.360550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[155391, 178330, 118221, 111671, 148152, 126147, 119127, 91537] avg=131072.0000 max=178330 max_rank=1 min=91537 min_rank=7 imbalance_ratio=1.360550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[142761, 102162, 66334, 151030, 123653, 210377, 100046, 152213] avg=131072.0000 max=210377 max_rank=5 min=66334 min_rank=2 imbalance_ratio=1.605049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[155391, 178330, 118221, 111671, 148152, 126147, 119127, 91537] avg=131072.0000 max=178330 max_rank=1 min=91537 min_rank=7 imbalance_ratio=1.360550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] avg=131072.0000 max=132384 max_rank=0 min=123844 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=77993 max_load_reduction_pct=0.370730 moved_to_replicas=231133 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[142761, 102162, 66334, 151030, 123653, 210377, 100046, 152213] avg=131072.0000 max=210377 max_rank=5 min=66334 min_rank=2 imbalance_ratio=1.605049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] avg=131072.0000 max=132384 max_rank=0 min=123844 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=77993 max_load_reduction_pct=0.370730 moved_to_replicas=231133 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] avg=131072.0000 max=132384 max_rank=2 min=126396 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45946 max_load_reduction_pct=0.257646 moved_to_replicas=170422 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] avg=131072.0000 max=132384 max_rank=2 min=126396 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45946 max_load_reduction_pct=0.257646 moved_to_replicas=170422 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[155391, 178330, 118221, 111671, 148152, 126147, 119127, 91537] avg=131072.0000 max=178330 max_rank=1 min=91537 min_rank=7 imbalance_ratio=1.360550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] avg=131072.0000 max=132384 max_rank=0 min=123844 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=77993 max_load_reduction_pct=0.370730 moved_to_replicas=231133 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] avg=131072.0000 max=132384 max_rank=2 min=126396 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45946 max_load_reduction_pct=0.257646 moved_to_replicas=170422 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[132384, 132381, 130434, 132383, 123844, 132384, 132382, 132384] avg=131072.0000 max=132384 max_rank=0 min=123844 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=77993 max_load_reduction_pct=0.370730 moved_to_replicas=231133 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[155391, 178330, 118221, 111671, 148152, 126147, 119127, 91537] avg=131072.0000 max=178330 max_rank=1 min=91537 min_rank=7 imbalance_ratio=1.360550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] avg=131072.0000 max=132384 max_rank=2 min=126396 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45946 max_load_reduction_pct=0.257646 moved_to_replicas=170422 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[155391, 178330, 118221, 111671, 148152, 126147, 119127, 91537] avg=131072.0000 max=178330 max_rank=1 min=91537 min_rank=7 imbalance_ratio=1.360550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[188021, 145380, 118945, 129741, 87230, 106740, 171117, 101402] avg=131072.0000 max=188021 max_rank=0 min=87230 min_rank=4 imbalance_ratio=1.434486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[188021, 145380, 118945, 129741, 87230, 106740, 171117, 101402] avg=131072.0000 max=188021 max_rank=0 min=87230 min_rank=4 imbalance_ratio=1.434486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[188021, 145380, 118945, 129741, 87230, 106740, 171117, 101402] avg=131072.0000 max=188021 max_rank=0 min=87230 min_rank=4 imbalance_ratio=1.434486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[155391, 178330, 118221, 111671, 148152, 126147, 119127, 91537] avg=131072.0000 max=178330 max_rank=1 min=91537 min_rank=7 imbalance_ratio=1.360550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] avg=131072.0000 max=132384 max_rank=2 min=126396 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45946 max_load_reduction_pct=0.257646 moved_to_replicas=170422 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[155391, 178330, 118221, 111671, 148152, 126147, 119127, 91537] avg=131072.0000 max=178330 max_rank=1 min=91537 min_rank=7 imbalance_ratio=1.360550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] avg=131072.0000 max=132384 max_rank=2 min=126396 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45946 max_load_reduction_pct=0.257646 moved_to_replicas=170422 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126122 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=55638 max_load_reduction_pct=0.295914 moved_to_replicas=220024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126122 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=55638 max_load_reduction_pct=0.295914 moved_to_replicas=220024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[188021, 145380, 118945, 129741, 87230, 106740, 171117, 101402] avg=131072.0000 max=188021 max_rank=0 min=87230 min_rank=4 imbalance_ratio=1.434486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126122 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=55638 max_load_reduction_pct=0.295914 moved_to_replicas=220024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] avg=131072.0000 max=132384 max_rank=2 min=126396 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45946 max_load_reduction_pct=0.257646 moved_to_replicas=170422 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[132383, 132382, 132384, 126396, 132383, 132292, 127972, 132384] avg=131072.0000 max=132384 max_rank=2 min=126396 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=45946 max_load_reduction_pct=0.257646 moved_to_replicas=170422 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126122 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=55638 max_load_reduction_pct=0.295914 moved_to_replicas=220024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[188021, 145380, 118945, 129741, 87230, 106740, 171117, 101402] avg=131072.0000 max=188021 max_rank=0 min=87230 min_rank=4 imbalance_ratio=1.434486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[188021, 145380, 118945, 129741, 87230, 106740, 171117, 101402] avg=131072.0000 max=188021 max_rank=0 min=87230 min_rank=4 imbalance_ratio=1.434486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[81209, 197468, 156183, 114342, 121168, 102064, 174559, 101583] avg=131072.0000 max=197468 max_rank=1 min=81209 min_rank=0 imbalance_ratio=1.506561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[81209, 197468, 156183, 114342, 121168, 102064, 174559, 101583] avg=131072.0000 max=197468 max_rank=1 min=81209 min_rank=0 imbalance_ratio=1.506561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[81209, 197468, 156183, 114342, 121168, 102064, 174559, 101583] avg=131072.0000 max=197468 max_rank=1 min=81209 min_rank=0 imbalance_ratio=1.506561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[188021, 145380, 118945, 129741, 87230, 106740, 171117, 101402] avg=131072.0000 max=188021 max_rank=0 min=87230 min_rank=4 imbalance_ratio=1.434486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[188021, 145380, 118945, 129741, 87230, 106740, 171117, 101402] avg=131072.0000 max=188021 max_rank=0 min=87230 min_rank=4 imbalance_ratio=1.434486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126122 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=55638 max_load_reduction_pct=0.295914 moved_to_replicas=220024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126122 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=55638 max_load_reduction_pct=0.295914 moved_to_replicas=220024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] avg=131072.0000 max=132384 max_rank=3 min=126927 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.329593 moved_to_replicas=291056 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[81209, 197468, 156183, 114342, 121168, 102064, 174559, 101583] avg=131072.0000 max=197468 max_rank=1 min=81209 min_rank=0 imbalance_ratio=1.506561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] avg=131072.0000 max=132384 max_rank=3 min=126927 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.329593 moved_to_replicas=291056 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] avg=131072.0000 max=132384 max_rank=3 min=126927 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.329593 moved_to_replicas=291056 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126122 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=55638 max_load_reduction_pct=0.295914 moved_to_replicas=220024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[132383, 132383, 130798, 129741, 126122, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126122 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=55638 max_load_reduction_pct=0.295914 moved_to_replicas=220024 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] avg=131072.0000 max=132384 max_rank=3 min=126927 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.329593 moved_to_replicas=291056 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[81209, 197468, 156183, 114342, 121168, 102064, 174559, 101583] avg=131072.0000 max=197468 max_rank=1 min=81209 min_rank=0 imbalance_ratio=1.506561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[81209, 197468, 156183, 114342, 121168, 102064, 174559, 101583] avg=131072.0000 max=197468 max_rank=1 min=81209 min_rank=0 imbalance_ratio=1.506561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[100459, 110532, 117072, 165378, 90197, 214568, 151303, 99067] avg=131072.0000 max=214568 max_rank=5 min=90197 min_rank=4 imbalance_ratio=1.637024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[100459, 110532, 117072, 165378, 90197, 214568, 151303, 99067] avg=131072.0000 max=214568 max_rank=5 min=90197 min_rank=4 imbalance_ratio=1.637024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[100459, 110532, 117072, 165378, 90197, 214568, 151303, 99067] avg=131072.0000 max=214568 max_rank=5 min=90197 min_rank=4 imbalance_ratio=1.637024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[81209, 197468, 156183, 114342, 121168, 102064, 174559, 101583] avg=131072.0000 max=197468 max_rank=1 min=81209 min_rank=0 imbalance_ratio=1.506561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] avg=131072.0000 max=132384 max_rank=3 min=126927 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.329593 moved_to_replicas=291056 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[81209, 197468, 156183, 114342, 121168, 102064, 174559, 101583] avg=131072.0000 max=197468 max_rank=1 min=81209 min_rank=0 imbalance_ratio=1.506561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] avg=131072.0000 max=132384 max_rank=5 min=127274 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=82184 max_load_reduction_pct=0.383021 moved_to_replicas=275692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] avg=131072.0000 max=132384 max_rank=3 min=126927 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.329593 moved_to_replicas=291056 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[100459, 110532, 117072, 165378, 90197, 214568, 151303, 99067] avg=131072.0000 max=214568 max_rank=5 min=90197 min_rank=4 imbalance_ratio=1.637024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] avg=131072.0000 max=132384 max_rank=5 min=127274 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=82184 max_load_reduction_pct=0.383021 moved_to_replicas=275692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] avg=131072.0000 max=132384 max_rank=5 min=127274 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=82184 max_load_reduction_pct=0.383021 moved_to_replicas=275692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] avg=131072.0000 max=132384 max_rank=3 min=126927 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.329593 moved_to_replicas=291056 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[127353, 132383, 132382, 132384, 126927, 132383, 132381, 132383] avg=131072.0000 max=132384 max_rank=3 min=126927 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65084 max_load_reduction_pct=0.329593 moved_to_replicas=291056 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] avg=131072.0000 max=132384 max_rank=5 min=127274 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=82184 max_load_reduction_pct=0.383021 moved_to_replicas=275692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[100459, 110532, 117072, 165378, 90197, 214568, 151303, 99067] avg=131072.0000 max=214568 max_rank=5 min=90197 min_rank=4 imbalance_ratio=1.637024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[162344, 44154, 142571, 231831, 129961, 132816, 80865, 124034] avg=131072.0000 max=231831 max_rank=3 min=44154 min_rank=1 imbalance_ratio=1.768730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[100459, 110532, 117072, 165378, 90197, 214568, 151303, 99067] avg=131072.0000 max=214568 max_rank=5 min=90197 min_rank=4 imbalance_ratio=1.637024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[162344, 44154, 142571, 231831, 129961, 132816, 80865, 124034] avg=131072.0000 max=231831 max_rank=3 min=44154 min_rank=1 imbalance_ratio=1.768730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[162344, 44154, 142571, 231831, 129961, 132816, 80865, 124034] avg=131072.0000 max=231831 max_rank=3 min=44154 min_rank=1 imbalance_ratio=1.768730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[100459, 110532, 117072, 165378, 90197, 214568, 151303, 99067] avg=131072.0000 max=214568 max_rank=5 min=90197 min_rank=4 imbalance_ratio=1.637024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] avg=131072.0000 max=132384 max_rank=5 min=127274 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=82184 max_load_reduction_pct=0.383021 moved_to_replicas=275692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[100459, 110532, 117072, 165378, 90197, 214568, 151303, 99067] avg=131072.0000 max=214568 max_rank=5 min=90197 min_rank=4 imbalance_ratio=1.637024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] avg=131072.0000 max=132384 max_rank=2 min=124467 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=99447 max_load_reduction_pct=0.428963 moved_to_replicas=314583 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] avg=131072.0000 max=132384 max_rank=5 min=127274 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=82184 max_load_reduction_pct=0.383021 moved_to_replicas=275692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[162344, 44154, 142571, 231831, 129961, 132816, 80865, 124034] avg=131072.0000 max=231831 max_rank=3 min=44154 min_rank=1 imbalance_ratio=1.768730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] avg=131072.0000 max=132384 max_rank=2 min=124467 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=99447 max_load_reduction_pct=0.428963 moved_to_replicas=314583 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] avg=131072.0000 max=132384 max_rank=2 min=124467 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=99447 max_load_reduction_pct=0.428963 moved_to_replicas=314583 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] avg=131072.0000 max=132384 max_rank=5 min=127274 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=82184 max_load_reduction_pct=0.383021 moved_to_replicas=275692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[132381, 132382, 127274, 132383, 128060, 132384, 132383, 131329] avg=131072.0000 max=132384 max_rank=5 min=127274 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=82184 max_load_reduction_pct=0.383021 moved_to_replicas=275692 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] avg=131072.0000 max=132384 max_rank=2 min=124467 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=99447 max_load_reduction_pct=0.428963 moved_to_replicas=314583 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[162344, 44154, 142571, 231831, 129961, 132816, 80865, 124034] avg=131072.0000 max=231831 max_rank=3 min=44154 min_rank=1 imbalance_ratio=1.768730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[116262, 135468, 97780, 82834, 108965, 195859, 131472, 179936] avg=131072.0000 max=195859 max_rank=5 min=82834 min_rank=3 imbalance_ratio=1.494286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[162344, 44154, 142571, 231831, 129961, 132816, 80865, 124034] avg=131072.0000 max=231831 max_rank=3 min=44154 min_rank=1 imbalance_ratio=1.768730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[116262, 135468, 97780, 82834, 108965, 195859, 131472, 179936] avg=131072.0000 max=195859 max_rank=5 min=82834 min_rank=3 imbalance_ratio=1.494286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[116262, 135468, 97780, 82834, 108965, 195859, 131472, 179936] avg=131072.0000 max=195859 max_rank=5 min=82834 min_rank=3 imbalance_ratio=1.494286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] avg=131072.0000 max=132384 max_rank=2 min=124467 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=99447 max_load_reduction_pct=0.428963 moved_to_replicas=314583 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[162344, 44154, 142571, 231831, 129961, 132816, 80865, 124034] avg=131072.0000 max=231831 max_rank=3 min=44154 min_rank=1 imbalance_ratio=1.768730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] avg=131072.0000 max=132384 max_rank=4 min=127517 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63475 max_load_reduction_pct=0.324085 moved_to_replicas=226097 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] avg=131072.0000 max=132384 max_rank=4 min=127517 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63475 max_load_reduction_pct=0.324085 moved_to_replicas=226097 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] avg=131072.0000 max=132384 max_rank=2 min=124467 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=99447 max_load_reduction_pct=0.428963 moved_to_replicas=314583 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[116262, 135468, 97780, 82834, 108965, 195859, 131472, 179936] avg=131072.0000 max=195859 max_rank=5 min=82834 min_rank=3 imbalance_ratio=1.494286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] avg=131072.0000 max=132384 max_rank=4 min=127517 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63475 max_load_reduction_pct=0.324085 moved_to_replicas=226097 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[162344, 44154, 142571, 231831, 129961, 132816, 80865, 124034] avg=131072.0000 max=231831 max_rank=3 min=44154 min_rank=1 imbalance_ratio=1.768730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] avg=131072.0000 max=132384 max_rank=2 min=124467 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=99447 max_load_reduction_pct=0.428963 moved_to_replicas=314583 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[116262, 135468, 97780, 82834, 108965, 195859, 131472, 179936] avg=131072.0000 max=195859 max_rank=5 min=82834 min_rank=3 imbalance_ratio=1.494286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] avg=131072.0000 max=132384 max_rank=4 min=127517 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63475 max_load_reduction_pct=0.324085 moved_to_replicas=226097 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[176523, 133796, 94700, 63819, 162027, 119620, 156419, 141672] avg=131072.0000 max=176523 max_rank=0 min=63819 min_rank=3 imbalance_ratio=1.346764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[176523, 133796, 94700, 63819, 162027, 119620, 156419, 141672] avg=131072.0000 max=176523 max_rank=0 min=63819 min_rank=3 imbalance_ratio=1.346764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[132383, 132383, 132384, 132383, 129961, 132383, 132232, 124467] avg=131072.0000 max=132384 max_rank=2 min=124467 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=99447 max_load_reduction_pct=0.428963 moved_to_replicas=314583 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[116262, 135468, 97780, 82834, 108965, 195859, 131472, 179936] avg=131072.0000 max=195859 max_rank=5 min=82834 min_rank=3 imbalance_ratio=1.494286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[176523, 133796, 94700, 63819, 162027, 119620, 156419, 141672] avg=131072.0000 max=176523 max_rank=0 min=63819 min_rank=3 imbalance_ratio=1.346764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] avg=131072.0000 max=132384 max_rank=4 min=127517 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63475 max_load_reduction_pct=0.324085 moved_to_replicas=226097 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[116262, 135468, 97780, 82834, 108965, 195859, 131472, 179936] avg=131072.0000 max=195859 max_rank=5 min=82834 min_rank=3 imbalance_ratio=1.494286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127162 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44140 max_load_reduction_pct=0.250052 moved_to_replicas=220270 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127162 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44140 max_load_reduction_pct=0.250052 moved_to_replicas=220270 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] avg=131072.0000 max=132384 max_rank=4 min=127517 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63475 max_load_reduction_pct=0.324085 moved_to_replicas=226097 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127162 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44140 max_load_reduction_pct=0.250052 moved_to_replicas=220270 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[176523, 133796, 94700, 63819, 162027, 119620, 156419, 141672] avg=131072.0000 max=176523 max_rank=0 min=63819 min_rank=3 imbalance_ratio=1.346764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[116262, 135468, 97780, 82834, 108965, 195859, 131472, 179936] avg=131072.0000 max=195859 max_rank=5 min=82834 min_rank=3 imbalance_ratio=1.494286 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] avg=131072.0000 max=132384 max_rank=4 min=127517 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63475 max_load_reduction_pct=0.324085 moved_to_replicas=226097 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[176523, 133796, 94700, 63819, 162027, 119620, 156419, 141672] avg=131072.0000 max=176523 max_rank=0 min=63819 min_rank=3 imbalance_ratio=1.346764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127162 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44140 max_load_reduction_pct=0.250052 moved_to_replicas=220270 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[133642, 160271, 136231, 103082, 184166, 142916, 76303, 111965] avg=131072.0000 max=184166 max_rank=4 min=76303 min_rank=6 imbalance_ratio=1.405075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[133642, 160271, 136231, 103082, 184166, 142916, 76303, 111965] avg=131072.0000 max=184166 max_rank=4 min=76303 min_rank=6 imbalance_ratio=1.405075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[131439, 132383, 128615, 127517, 132384, 132383, 131472, 132383] avg=131072.0000 max=132384 max_rank=4 min=127517 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63475 max_load_reduction_pct=0.324085 moved_to_replicas=226097 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[176523, 133796, 94700, 63819, 162027, 119620, 156419, 141672] avg=131072.0000 max=176523 max_rank=0 min=63819 min_rank=3 imbalance_ratio=1.346764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[133642, 160271, 136231, 103082, 184166, 142916, 76303, 111965] avg=131072.0000 max=184166 max_rank=4 min=76303 min_rank=6 imbalance_ratio=1.405075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127162 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44140 max_load_reduction_pct=0.250052 moved_to_replicas=220270 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[176523, 133796, 94700, 63819, 162027, 119620, 156419, 141672] avg=131072.0000 max=176523 max_rank=0 min=63819 min_rank=3 imbalance_ratio=1.346764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] avg=131072.0000 max=132383 max_rank=0 min=127035 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=51783 max_load_reduction_pct=0.281176 moved_to_replicas=197230 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] avg=131072.0000 max=132383 max_rank=0 min=127035 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=51783 max_load_reduction_pct=0.281176 moved_to_replicas=197230 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] avg=131072.0000 max=132383 max_rank=0 min=127035 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=51783 max_load_reduction_pct=0.281176 moved_to_replicas=197230 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[133642, 160271, 136231, 103082, 184166, 142916, 76303, 111965] avg=131072.0000 max=184166 max_rank=4 min=76303 min_rank=6 imbalance_ratio=1.405075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127162 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44140 max_load_reduction_pct=0.250052 moved_to_replicas=220270 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[176523, 133796, 94700, 63819, 162027, 119620, 156419, 141672] avg=131072.0000 max=176523 max_rank=0 min=63819 min_rank=3 imbalance_ratio=1.346764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127162 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44140 max_load_reduction_pct=0.250052 moved_to_replicas=220270 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[133642, 160271, 136231, 103082, 184166, 142916, 76303, 111965] avg=131072.0000 max=184166 max_rank=4 min=76303 min_rank=6 imbalance_ratio=1.405075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] avg=131072.0000 max=132383 max_rank=0 min=127035 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=51783 max_load_reduction_pct=0.281176 moved_to_replicas=197230 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[42064, 138697, 156989, 135635, 103980, 123408, 129096, 218707] avg=131072.0000 max=218707 max_rank=7 min=42064 min_rank=0 imbalance_ratio=1.668602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[42064, 138697, 156989, 135635, 103980, 123408, 129096, 218707] avg=131072.0000 max=218707 max_rank=7 min=42064 min_rank=0 imbalance_ratio=1.668602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[132382, 132383, 130591, 127162, 132383, 128909, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127162 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=44140 max_load_reduction_pct=0.250052 moved_to_replicas=220270 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[42064, 138697, 156989, 135635, 103980, 123408, 129096, 218707] avg=131072.0000 max=218707 max_rank=7 min=42064 min_rank=0 imbalance_ratio=1.668602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] avg=131072.0000 max=132383 max_rank=0 min=127035 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=51783 max_load_reduction_pct=0.281176 moved_to_replicas=197230 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[133642, 160271, 136231, 103082, 184166, 142916, 76303, 111965] avg=131072.0000 max=184166 max_rank=4 min=76303 min_rank=6 imbalance_ratio=1.405075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] avg=131072.0000 max=132384 max_rank=1 min=129096 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86323 max_load_reduction_pct=0.394697 moved_to_replicas=264399 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] avg=131072.0000 max=132384 max_rank=1 min=129096 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86323 max_load_reduction_pct=0.394697 moved_to_replicas=264399 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] avg=131072.0000 max=132384 max_rank=1 min=129096 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86323 max_load_reduction_pct=0.394697 moved_to_replicas=264399 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[42064, 138697, 156989, 135635, 103980, 123408, 129096, 218707] avg=131072.0000 max=218707 max_rank=7 min=42064 min_rank=0 imbalance_ratio=1.668602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[133642, 160271, 136231, 103082, 184166, 142916, 76303, 111965] avg=131072.0000 max=184166 max_rank=4 min=76303 min_rank=6 imbalance_ratio=1.405075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[133642, 160271, 136231, 103082, 184166, 142916, 76303, 111965] avg=131072.0000 max=184166 max_rank=4 min=76303 min_rank=6 imbalance_ratio=1.405075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] avg=131072.0000 max=132383 max_rank=0 min=127035 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=51783 max_load_reduction_pct=0.281176 moved_to_replicas=197230 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[42064, 138697, 156989, 135635, 103980, 123408, 129096, 218707] avg=131072.0000 max=218707 max_rank=7 min=42064 min_rank=0 imbalance_ratio=1.668602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] avg=131072.0000 max=132384 max_rank=1 min=129096 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86323 max_load_reduction_pct=0.394697 moved_to_replicas=264399 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[108378, 152538, 113179, 110253, 91831, 190330, 133286, 148781] avg=131072.0000 max=190330 max_rank=5 min=91831 min_rank=4 imbalance_ratio=1.452103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] avg=131072.0000 max=132383 max_rank=0 min=127035 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=51783 max_load_reduction_pct=0.281176 moved_to_replicas=197230 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[108378, 152538, 113179, 110253, 91831, 190330, 133286, 148781] avg=131072.0000 max=190330 max_rank=5 min=91831 min_rank=4 imbalance_ratio=1.452103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[132383, 132383, 132383, 127244, 132383, 132383, 132382, 127035] avg=131072.0000 max=132383 max_rank=0 min=127035 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=51783 max_load_reduction_pct=0.281176 moved_to_replicas=197230 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[108378, 152538, 113179, 110253, 91831, 190330, 133286, 148781] avg=131072.0000 max=190330 max_rank=5 min=91831 min_rank=4 imbalance_ratio=1.452103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] avg=131072.0000 max=132384 max_rank=1 min=129096 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86323 max_load_reduction_pct=0.394697 moved_to_replicas=264399 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[42064, 138697, 156989, 135635, 103980, 123408, 129096, 218707] avg=131072.0000 max=218707 max_rank=7 min=42064 min_rank=0 imbalance_ratio=1.668602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=125175 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57946 max_load_reduction_pct=0.304450 moved_to_replicas=197184 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=125175 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57946 max_load_reduction_pct=0.304450 moved_to_replicas=197184 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=125175 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57946 max_load_reduction_pct=0.304450 moved_to_replicas=197184 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[108378, 152538, 113179, 110253, 91831, 190330, 133286, 148781] avg=131072.0000 max=190330 max_rank=5 min=91831 min_rank=4 imbalance_ratio=1.452103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[42064, 138697, 156989, 135635, 103980, 123408, 129096, 218707] avg=131072.0000 max=218707 max_rank=7 min=42064 min_rank=0 imbalance_ratio=1.668602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] avg=131072.0000 max=132384 max_rank=1 min=129096 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86323 max_load_reduction_pct=0.394697 moved_to_replicas=264399 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[42064, 138697, 156989, 135635, 103980, 123408, 129096, 218707] avg=131072.0000 max=218707 max_rank=7 min=42064 min_rank=0 imbalance_ratio=1.668602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[108378, 152538, 113179, 110253, 91831, 190330, 133286, 148781] avg=131072.0000 max=190330 max_rank=5 min=91831 min_rank=4 imbalance_ratio=1.452103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=125175 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57946 max_load_reduction_pct=0.304450 moved_to_replicas=197184 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[140695, 160375, 130130, 133646, 142657, 70821, 130752, 139500] avg=131072.0000 max=160375 max_rank=1 min=70821 min_rank=5 imbalance_ratio=1.223564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] avg=131072.0000 max=132384 max_rank=1 min=129096 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86323 max_load_reduction_pct=0.394697 moved_to_replicas=264399 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[140695, 160375, 130130, 133646, 142657, 70821, 130752, 139500] avg=131072.0000 max=160375 max_rank=1 min=70821 min_rank=5 imbalance_ratio=1.223564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[140695, 160375, 130130, 133646, 142657, 70821, 130752, 139500] avg=131072.0000 max=160375 max_rank=1 min=70821 min_rank=5 imbalance_ratio=1.223564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[130129, 132384, 132382, 132383, 130098, 129721, 129096, 132383] avg=131072.0000 max=132384 max_rank=1 min=129096 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86323 max_load_reduction_pct=0.394697 moved_to_replicas=264399 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=125175 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57946 max_load_reduction_pct=0.304450 moved_to_replicas=197184 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[108378, 152538, 113179, 110253, 91831, 190330, 133286, 148781] avg=131072.0000 max=190330 max_rank=5 min=91831 min_rank=4 imbalance_ratio=1.452103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] avg=131072.0000 max=132384 max_rank=7 min=125779 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27991 max_load_reduction_pct=0.174535 moved_to_replicas=116058 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] avg=131072.0000 max=132384 max_rank=7 min=125779 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27991 max_load_reduction_pct=0.174535 moved_to_replicas=116058 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[140695, 160375, 130130, 133646, 142657, 70821, 130752, 139500] avg=131072.0000 max=160375 max_rank=1 min=70821 min_rank=5 imbalance_ratio=1.223564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] avg=131072.0000 max=132384 max_rank=7 min=125779 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27991 max_load_reduction_pct=0.174535 moved_to_replicas=116058 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[108378, 152538, 113179, 110253, 91831, 190330, 133286, 148781] avg=131072.0000 max=190330 max_rank=5 min=91831 min_rank=4 imbalance_ratio=1.452103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=125175 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57946 max_load_reduction_pct=0.304450 moved_to_replicas=197184 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[108378, 152538, 113179, 110253, 91831, 190330, 133286, 148781] avg=131072.0000 max=190330 max_rank=5 min=91831 min_rank=4 imbalance_ratio=1.452103 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[140695, 160375, 130130, 133646, 142657, 70821, 130752, 139500] avg=131072.0000 max=160375 max_rank=1 min=70821 min_rank=5 imbalance_ratio=1.223564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] avg=131072.0000 max=132384 max_rank=7 min=125779 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27991 max_load_reduction_pct=0.174535 moved_to_replicas=116058 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[88889, 46021, 158450, 137976, 133633, 188418, 133007, 162182] avg=131072.0000 max=188418 max_rank=5 min=46021 min_rank=1 imbalance_ratio=1.437515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[88889, 46021, 158450, 137976, 133633, 188418, 133007, 162182] avg=131072.0000 max=188418 max_rank=5 min=46021 min_rank=1 imbalance_ratio=1.437515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=125175 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57946 max_load_reduction_pct=0.304450 moved_to_replicas=197184 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[88889, 46021, 158450, 137976, 133633, 188418, 133007, 162182] avg=131072.0000 max=188418 max_rank=5 min=46021 min_rank=1 imbalance_ratio=1.437515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[132383, 132384, 129576, 131908, 125175, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=1 min=125175 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57946 max_load_reduction_pct=0.304450 moved_to_replicas=197184 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] avg=131072.0000 max=132384 max_rank=7 min=125779 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27991 max_load_reduction_pct=0.174535 moved_to_replicas=116058 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[140695, 160375, 130130, 133646, 142657, 70821, 130752, 139500] avg=131072.0000 max=160375 max_rank=1 min=70821 min_rank=5 imbalance_ratio=1.223564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=126155 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=56035 max_load_reduction_pct=0.297397 moved_to_replicas=243530 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=126155 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=56035 max_load_reduction_pct=0.297397 moved_to_replicas=243530 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[88889, 46021, 158450, 137976, 133633, 188418, 133007, 162182] avg=131072.0000 max=188418 max_rank=5 min=46021 min_rank=1 imbalance_ratio=1.437515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=126155 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=56035 max_load_reduction_pct=0.297397 moved_to_replicas=243530 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[140695, 160375, 130130, 133646, 142657, 70821, 130752, 139500] avg=131072.0000 max=160375 max_rank=1 min=70821 min_rank=5 imbalance_ratio=1.223564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] avg=131072.0000 max=132384 max_rank=7 min=125779 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27991 max_load_reduction_pct=0.174535 moved_to_replicas=116058 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[88889, 46021, 158450, 137976, 133633, 188418, 133007, 162182] avg=131072.0000 max=188418 max_rank=5 min=46021 min_rank=1 imbalance_ratio=1.437515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[140695, 160375, 130130, 133646, 142657, 70821, 130752, 139500] avg=131072.0000 max=160375 max_rank=1 min=70821 min_rank=5 imbalance_ratio=1.223564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=126155 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=56035 max_load_reduction_pct=0.297397 moved_to_replicas=243530 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[171394, 158965, 26146, 102771, 116251, 183615, 141260, 148174] avg=131072.0000 max=183615 max_rank=5 min=26146 min_rank=2 imbalance_ratio=1.400871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[171394, 158965, 26146, 102771, 116251, 183615, 141260, 148174] avg=131072.0000 max=183615 max_rank=5 min=26146 min_rank=2 imbalance_ratio=1.400871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[171394, 158965, 26146, 102771, 116251, 183615, 141260, 148174] avg=131072.0000 max=183615 max_rank=5 min=26146 min_rank=2 imbalance_ratio=1.400871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] avg=131072.0000 max=132384 max_rank=7 min=125779 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27991 max_load_reduction_pct=0.174535 moved_to_replicas=116058 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=126155 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=56035 max_load_reduction_pct=0.297397 moved_to_replicas=243530 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[132383, 132382, 130130, 132383, 132383, 125779, 130752, 132384] avg=131072.0000 max=132384 max_rank=7 min=125779 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=27991 max_load_reduction_pct=0.174535 moved_to_replicas=116058 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125129 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=51231 max_load_reduction_pct=0.279013 moved_to_replicas=311063 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[88889, 46021, 158450, 137976, 133633, 188418, 133007, 162182] avg=131072.0000 max=188418 max_rank=5 min=46021 min_rank=1 imbalance_ratio=1.437515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125129 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=51231 max_load_reduction_pct=0.279013 moved_to_replicas=311063 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[171394, 158965, 26146, 102771, 116251, 183615, 141260, 148174] avg=131072.0000 max=183615 max_rank=5 min=26146 min_rank=2 imbalance_ratio=1.400871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125129 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=51231 max_load_reduction_pct=0.279013 moved_to_replicas=311063 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[88889, 46021, 158450, 137976, 133633, 188418, 133007, 162182] avg=131072.0000 max=188418 max_rank=5 min=46021 min_rank=1 imbalance_ratio=1.437515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[171394, 158965, 26146, 102771, 116251, 183615, 141260, 148174] avg=131072.0000 max=183615 max_rank=5 min=26146 min_rank=2 imbalance_ratio=1.400871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=126155 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=56035 max_load_reduction_pct=0.297397 moved_to_replicas=243530 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[88889, 46021, 158450, 137976, 133633, 188418, 133007, 162182] avg=131072.0000 max=188418 max_rank=5 min=46021 min_rank=1 imbalance_ratio=1.437515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125129 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=51231 max_load_reduction_pct=0.279013 moved_to_replicas=311063 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[115993, 97475, 118025, 106760, 103113, 191232, 145490, 170488] avg=131072.0000 max=191232 max_rank=5 min=97475 min_rank=1 imbalance_ratio=1.458984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[115993, 97475, 118025, 106760, 103113, 191232, 145490, 170488] avg=131072.0000 max=191232 max_rank=5 min=97475 min_rank=1 imbalance_ratio=1.458984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[115993, 97475, 118025, 106760, 103113, 191232, 145490, 170488] avg=131072.0000 max=191232 max_rank=5 min=97475 min_rank=1 imbalance_ratio=1.458984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=126155 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=56035 max_load_reduction_pct=0.297397 moved_to_replicas=243530 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125129 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=51231 max_load_reduction_pct=0.279013 moved_to_replicas=311063 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[126155, 128126, 132381, 132383, 132383, 132382, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=126155 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=56035 max_load_reduction_pct=0.297397 moved_to_replicas=243530 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=3 min=127223 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=58848 max_load_reduction_pct=0.307731 moved_to_replicas=219120 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[171394, 158965, 26146, 102771, 116251, 183615, 141260, 148174] avg=131072.0000 max=183615 max_rank=5 min=26146 min_rank=2 imbalance_ratio=1.400871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=3 min=127223 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=58848 max_load_reduction_pct=0.307731 moved_to_replicas=219120 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[115993, 97475, 118025, 106760, 103113, 191232, 145490, 170488] avg=131072.0000 max=191232 max_rank=5 min=97475 min_rank=1 imbalance_ratio=1.458984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=3 min=127223 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=58848 max_load_reduction_pct=0.307731 moved_to_replicas=219120 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[115993, 97475, 118025, 106760, 103113, 191232, 145490, 170488] avg=131072.0000 max=191232 max_rank=5 min=97475 min_rank=1 imbalance_ratio=1.458984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[171394, 158965, 26146, 102771, 116251, 183615, 141260, 148174] avg=131072.0000 max=183615 max_rank=5 min=26146 min_rank=2 imbalance_ratio=1.400871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125129 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=51231 max_load_reduction_pct=0.279013 moved_to_replicas=311063 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[171394, 158965, 26146, 102771, 116251, 183615, 141260, 148174] avg=131072.0000 max=183615 max_rank=5 min=26146 min_rank=2 imbalance_ratio=1.400871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=3 min=127223 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=58848 max_load_reduction_pct=0.307731 moved_to_replicas=219120 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=3 min=127223 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=58848 max_load_reduction_pct=0.307731 moved_to_replicas=219120 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125129 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=51231 max_load_reduction_pct=0.279013 moved_to_replicas=311063 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[132384, 132381, 129150, 132384, 125129, 132384, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125129 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=51231 max_load_reduction_pct=0.279013 moved_to_replicas=311063 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[115993, 97475, 118025, 106760, 103113, 191232, 145490, 170488] avg=131072.0000 max=191232 max_rank=5 min=97475 min_rank=1 imbalance_ratio=1.458984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[115993, 97475, 118025, 106760, 103113, 191232, 145490, 170488] avg=131072.0000 max=191232 max_rank=5 min=97475 min_rank=1 imbalance_ratio=1.458984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=3 min=127223 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=58848 max_load_reduction_pct=0.307731 moved_to_replicas=219120 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[115993, 97475, 118025, 106760, 103113, 191232, 145490, 170488] avg=131072.0000 max=191232 max_rank=5 min=97475 min_rank=1 imbalance_ratio=1.458984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=3 min=127223 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=58848 max_load_reduction_pct=0.307731 moved_to_replicas=219120 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[132382, 129811, 127223, 132384, 129626, 132383, 132383, 132384] avg=131072.0000 max=132384 max_rank=3 min=127223 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=58848 max_load_reduction_pct=0.307731 moved_to_replicas=219120 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=4175301 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1027456 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=4170466 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1034368 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=4210193 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=996352 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=4198789 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1005696 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=4173655 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1031680 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=4203686 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=999168 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=4219551 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=985600 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=4202791 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1002112 peak_demand=132384 required_capacity=1.010010 + [2026-08-30 05:17:52] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 575.3 | throughput per GPU (TFLOP/s/GPU): 579.5 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.060068E+01 | loss scale: 1.0 | grad norm: 216.436 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=163840 expected_recv=256616 expected_overflow=0 exact_recv_per_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=163840 expected_recv=259911 expected_overflow=0 exact_recv_per_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=163840 expected_recv=260187 expected_overflow=0 exact_recv_per_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=163840 expected_recv=261374 expected_overflow=0 exact_recv_per_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=163840 expected_recv=256616 actual_valid_recv=258816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=163840 expected_recv=259911 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=163840 expected_recv=260187 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=258816 slack_slots=68864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=163840 expected_recv=261374 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=163840 expected_recv=261493 expected_overflow=0 exact_recv_per_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=163840 expected_recv=250603 expected_overflow=0 exact_recv_per_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=163840 expected_recv=261493 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=163840 expected_recv=261226 expected_overflow=0 exact_recv_per_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=163840 expected_recv=250603 actual_valid_recv=253312 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=163840 expected_recv=261226 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=253312 slack_slots=74368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=163840 expected_recv=259706 expected_overflow=0 exact_recv_per_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=163840 expected_recv=263710 expected_overflow=0 exact_recv_per_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=163840 expected_recv=259706 actual_valid_recv=262656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=163840 expected_recv=263710 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=262656 slack_slots=65024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=163840 expected_recv=261028 expected_overflow=0 exact_recv_per_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=163840 expected_recv=253646 expected_overflow=0 exact_recv_per_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=163840 expected_recv=261028 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=163840 expected_recv=253646 actual_valid_recv=256768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=256768 slack_slots=70912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=163840 expected_recv=259009 expected_overflow=0 exact_recv_per_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=163840 expected_recv=261860 expected_overflow=0 exact_recv_per_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=163840 expected_recv=259009 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=163840 expected_recv=261860 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=163840 expected_recv=252451 expected_overflow=0 exact_recv_per_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=163840 expected_recv=252451 actual_valid_recv=255360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=163840 expected_recv=266791 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=255360 slack_slots=72320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=163840 expected_recv=266790 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=163840 expected_recv=266791 actual_valid_recv=269312 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=269312 slack_slots=58368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=163840 expected_recv=266790 actual_valid_recv=269696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=163840 expected_recv=265105 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=269696 slack_slots=57984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=163840 expected_recv=255891 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=163840 expected_recv=265105 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=163840 expected_recv=266791 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=163840 expected_recv=266790 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=163840 expected_recv=255891 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=163840 expected_recv=266791 actual_valid_recv=269312 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=163840 expected_recv=266790 actual_valid_recv=269568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=163840 expected_recv=266792 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=259456 slack_slots=68224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=269312 slack_slots=58368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=269568 slack_slots=58112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=163840 expected_recv=266792 actual_valid_recv=269696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262155 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=269696 slack_slots=57984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262155 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=163840 expected_recv=260759 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=163840 expected_recv=260759 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=163840 expected_recv=242202 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=163840 expected_recv=250409 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=163840 expected_recv=250409 actual_valid_recv=253312 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=163840 expected_recv=242202 actual_valid_recv=245376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=163840 expected_recv=263756 expected_overflow=0 exact_recv_per_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=253312 slack_slots=74368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=245376 slack_slots=82304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=163840 expected_recv=263756 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262450 expected_overflow=0 exact_recv_per_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262450 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=163840 expected_recv=258226 expected_overflow=0 exact_recv_per_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262920 expected_overflow=0 exact_recv_per_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=163840 expected_recv=258226 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262920 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262651 expected_overflow=0 exact_recv_per_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=265856 slack_slots=61824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262651 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265472 slack_slots=62208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=163840 expected_recv=253867 expected_overflow=0 exact_recv_per_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=163840 expected_recv=253867 actual_valid_recv=256256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=256256 slack_slots=71424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=163840 expected_recv=254710 expected_overflow=0 exact_recv_per_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=163840 expected_recv=255499 expected_overflow=0 exact_recv_per_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=163840 expected_recv=259216 expected_overflow=0 exact_recv_per_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262356 expected_overflow=0 exact_recv_per_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=163840 expected_recv=254710 actual_valid_recv=256768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=163840 expected_recv=255499 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=163840 expected_recv=259216 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262356 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=256768 slack_slots=70912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=258304 slack_slots=69376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265472 slack_slots=62208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262306 expected_overflow=0 exact_recv_per_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262306 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=163840 expected_recv=264590 expected_overflow=0 exact_recv_per_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=163840 expected_recv=263728 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=163840 expected_recv=256483 expected_overflow=0 exact_recv_per_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=163840 expected_recv=264590 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=163840 expected_recv=263728 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=163840 expected_recv=256483 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262080 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=259200 slack_slots=68480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267904 slack_slots=59776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262080 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=163840 expected_recv=257304 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=163840 expected_recv=257304 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=163840 expected_recv=254975 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=163840 expected_recv=254975 actual_valid_recv=258048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=163840 expected_recv=255173 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=258048 slack_slots=69632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=163840 expected_recv=255173 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=163840 expected_recv=256237 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=257792 slack_slots=69888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=163840 expected_recv=262612 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=163840 expected_recv=256237 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=163840 expected_recv=264519 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=259072 slack_slots=68608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=163840 expected_recv=262612 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=163840 expected_recv=264519 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=163840 expected_recv=254791 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=268160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=163840 expected_recv=254791 actual_valid_recv=257536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262417 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=268160 slack_slots=59520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=257536 slack_slots=70144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=163840 expected_recv=261853 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262417 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=163840 expected_recv=261853 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=163840 expected_recv=264064 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=163840 expected_recv=264064 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=163840 expected_recv=253342 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=163840 expected_recv=261851 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=163840 expected_recv=253342 actual_valid_recv=255872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=163840 expected_recv=258209 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=163840 expected_recv=261851 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=255872 slack_slots=71808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262206 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=163840 expected_recv=258209 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262206 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=260480 slack_slots=67200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262188 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262188 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=163840 expected_recv=255365 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=163840 expected_recv=255365 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=163840 expected_recv=256602 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=257792 slack_slots=69888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=163840 expected_recv=263906 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=163840 expected_recv=256602 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=163840 expected_recv=263906 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=259328 slack_slots=68352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=163840 expected_recv=264332 expected_overflow=0 exact_recv_per_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=163840 expected_recv=264332 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=163840 expected_recv=263064 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=163840 expected_recv=263064 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265472 slack_slots=62208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=163840 expected_recv=253013 expected_overflow=0 exact_recv_per_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=163840 expected_recv=260582 expected_overflow=0 exact_recv_per_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=163840 expected_recv=253013 actual_valid_recv=256000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=163840 expected_recv=263915 expected_overflow=0 exact_recv_per_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=163840 expected_recv=260582 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=256000 slack_slots=71680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=163840 expected_recv=263915 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=163840 expected_recv=257886 expected_overflow=0 exact_recv_per_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=163840 expected_recv=257886 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=163840 expected_recv=258818 expected_overflow=0 exact_recv_per_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=163840 expected_recv=259566 expected_overflow=0 exact_recv_per_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[91964, 127804, 142069, 106907, 153990, 143846, 168463, 113533] avg=131072.0000 max=168463 max_rank=6 min=91964 min_rank=0 imbalance_ratio=1.285271 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=260352 slack_slots=67328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=163840 expected_recv=258818 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=163840 expected_recv=259729 expected_overflow=0 exact_recv_per_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=163840 expected_recv=259566 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[91964, 127804, 142069, 106907, 153990, 143846, 168463, 113533] avg=131072.0000 max=168463 max_rank=6 min=91964 min_rank=0 imbalance_ratio=1.285271 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=163840 expected_recv=259729 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=4 min=127804 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36079 max_load_reduction_pct=0.214166 moved_to_replicas=155333 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=4 min=127804 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36079 max_load_reduction_pct=0.214166 moved_to_replicas=155333 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[91964, 127804, 142069, 106907, 153990, 143846, 168463, 113533] avg=131072.0000 max=168463 max_rank=6 min=91964 min_rank=0 imbalance_ratio=1.285271 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[91964, 127804, 142069, 106907, 153990, 143846, 168463, 113533] avg=131072.0000 max=168463 max_rank=6 min=91964 min_rank=0 imbalance_ratio=1.285271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[91964, 127804, 142069, 106907, 153990, 143846, 168463, 113533] avg=131072.0000 max=168463 max_rank=6 min=91964 min_rank=0 imbalance_ratio=1.285271 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=163840 expected_recv=261816 expected_overflow=0 exact_recv_per_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[77895, 147133, 89513, 125342, 240119, 151195, 77618, 139761] avg=131072.0000 max=240119 max_rank=4 min=77618 min_rank=6 imbalance_ratio=1.831963 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=163840 expected_recv=261816 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=4 min=127804 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36079 max_load_reduction_pct=0.214166 moved_to_replicas=155333 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=4 min=127804 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36079 max_load_reduction_pct=0.214166 moved_to_replicas=155333 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[77895, 147133, 89513, 125342, 240119, 151195, 77618, 139761] avg=131072.0000 max=240119 max_rank=4 min=77618 min_rank=6 imbalance_ratio=1.831963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[91964, 127804, 142069, 106907, 153990, 143846, 168463, 113533] avg=131072.0000 max=168463 max_rank=6 min=91964 min_rank=0 imbalance_ratio=1.285271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=4 min=127804 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36079 max_load_reduction_pct=0.214166 moved_to_replicas=155333 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[91964, 127804, 142069, 106907, 153990, 143846, 168463, 113533] avg=131072.0000 max=168463 max_rank=6 min=91964 min_rank=0 imbalance_ratio=1.285271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=125342 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107735 max_load_reduction_pct=0.448673 moved_to_replicas=299192 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=125342 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107735 max_load_reduction_pct=0.448673 moved_to_replicas=299192 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[77895, 147133, 89513, 125342, 240119, 151195, 77618, 139761] avg=131072.0000 max=240119 max_rank=4 min=77618 min_rank=6 imbalance_ratio=1.831963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[77895, 147133, 89513, 125342, 240119, 151195, 77618, 139761] avg=131072.0000 max=240119 max_rank=4 min=77618 min_rank=6 imbalance_ratio=1.831963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=4 min=127804 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36079 max_load_reduction_pct=0.214166 moved_to_replicas=155333 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=4 min=127804 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36079 max_load_reduction_pct=0.214166 moved_to_replicas=155333 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[91964, 127804, 142069, 106907, 153990, 143846, 168463, 113533] avg=131072.0000 max=168463 max_rank=6 min=91964 min_rank=0 imbalance_ratio=1.285271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[77895, 147133, 89513, 125342, 240119, 151195, 77618, 139761] avg=131072.0000 max=240119 max_rank=4 min=77618 min_rank=6 imbalance_ratio=1.831963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[124053, 208690, 135652, 112809, 168448, 109018, 146773, 43133] avg=131072.0000 max=208690 max_rank=1 min=43133 min_rank=7 imbalance_ratio=1.592178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=125342 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107735 max_load_reduction_pct=0.448673 moved_to_replicas=299192 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=125342 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107735 max_load_reduction_pct=0.448673 moved_to_replicas=299192 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[124053, 208690, 135652, 112809, 168448, 109018, 146773, 43133] avg=131072.0000 max=208690 max_rank=1 min=43133 min_rank=7 imbalance_ratio=1.592178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[129902, 127804, 132382, 128956, 132384, 132383, 132383, 132382] avg=131072.0000 max=132384 max_rank=4 min=127804 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36079 max_load_reduction_pct=0.214166 moved_to_replicas=155333 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=125342 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107735 max_load_reduction_pct=0.448673 moved_to_replicas=299192 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] avg=131072.0000 max=132385 max_rank=5 min=126956 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=76305 max_load_reduction_pct=0.365638 moved_to_replicas=254920 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[77895, 147133, 89513, 125342, 240119, 151195, 77618, 139761] avg=131072.0000 max=240119 max_rank=4 min=77618 min_rank=6 imbalance_ratio=1.831963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[77895, 147133, 89513, 125342, 240119, 151195, 77618, 139761] avg=131072.0000 max=240119 max_rank=4 min=77618 min_rank=6 imbalance_ratio=1.831963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] avg=131072.0000 max=132385 max_rank=5 min=126956 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=76305 max_load_reduction_pct=0.365638 moved_to_replicas=254920 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[124053, 208690, 135652, 112809, 168448, 109018, 146773, 43133] avg=131072.0000 max=208690 max_rank=1 min=43133 min_rank=7 imbalance_ratio=1.592178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[124053, 208690, 135652, 112809, 168448, 109018, 146773, 43133] avg=131072.0000 max=208690 max_rank=1 min=43133 min_rank=7 imbalance_ratio=1.592178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=125342 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107735 max_load_reduction_pct=0.448673 moved_to_replicas=299192 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=125342 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107735 max_load_reduction_pct=0.448673 moved_to_replicas=299192 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[77895, 147133, 89513, 125342, 240119, 151195, 77618, 139761] avg=131072.0000 max=240119 max_rank=4 min=77618 min_rank=6 imbalance_ratio=1.831963 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[124053, 208690, 135652, 112809, 168448, 109018, 146773, 43133] avg=131072.0000 max=208690 max_rank=1 min=43133 min_rank=7 imbalance_ratio=1.592178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[91161, 91795, 220460, 41229, 159228, 139301, 170967, 134435] avg=131072.0000 max=220460 max_rank=2 min=41229 min_rank=3 imbalance_ratio=1.681976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] avg=131072.0000 max=132385 max_rank=5 min=126956 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=76305 max_load_reduction_pct=0.365638 moved_to_replicas=254920 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] avg=131072.0000 max=132385 max_rank=5 min=126956 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=76305 max_load_reduction_pct=0.365638 moved_to_replicas=254920 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[91161, 91795, 220460, 41229, 159228, 139301, 170967, 134435] avg=131072.0000 max=220460 max_rank=2 min=41229 min_rank=3 imbalance_ratio=1.681976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[129111, 132383, 132208, 125342, 132382, 132383, 132384, 132383] avg=131072.0000 max=132384 max_rank=6 min=125342 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=107735 max_load_reduction_pct=0.448673 moved_to_replicas=299192 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] avg=131072.0000 max=132385 max_rank=5 min=126956 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=76305 max_load_reduction_pct=0.365638 moved_to_replicas=254920 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] avg=131072.0000 max=132384 max_rank=5 min=126474 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=88076 max_load_reduction_pct=0.399510 moved_to_replicas=310941 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[124053, 208690, 135652, 112809, 168448, 109018, 146773, 43133] avg=131072.0000 max=208690 max_rank=1 min=43133 min_rank=7 imbalance_ratio=1.592178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[124053, 208690, 135652, 112809, 168448, 109018, 146773, 43133] avg=131072.0000 max=208690 max_rank=1 min=43133 min_rank=7 imbalance_ratio=1.592178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] avg=131072.0000 max=132384 max_rank=5 min=126474 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=88076 max_load_reduction_pct=0.399510 moved_to_replicas=310941 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[91161, 91795, 220460, 41229, 159228, 139301, 170967, 134435] avg=131072.0000 max=220460 max_rank=2 min=41229 min_rank=3 imbalance_ratio=1.681976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[91161, 91795, 220460, 41229, 159228, 139301, 170967, 134435] avg=131072.0000 max=220460 max_rank=2 min=41229 min_rank=3 imbalance_ratio=1.681976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] avg=131072.0000 max=132385 max_rank=5 min=126956 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=76305 max_load_reduction_pct=0.365638 moved_to_replicas=254920 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[149505, 185927, 133195, 186025, 149840, 1792, 85559, 156733] avg=131072.0000 max=186025 max_rank=3 min=1792 min_rank=5 imbalance_ratio=1.419258 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[124053, 208690, 135652, 112809, 168448, 109018, 146773, 43133] avg=131072.0000 max=208690 max_rank=1 min=43133 min_rank=7 imbalance_ratio=1.592178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[91161, 91795, 220460, 41229, 159228, 139301, 170967, 134435] avg=131072.0000 max=220460 max_rank=2 min=41229 min_rank=3 imbalance_ratio=1.681976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] avg=131072.0000 max=132385 max_rank=5 min=126956 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=76305 max_load_reduction_pct=0.365638 moved_to_replicas=254920 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] avg=131072.0000 max=132384 max_rank=5 min=126474 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=88076 max_load_reduction_pct=0.399510 moved_to_replicas=310941 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] avg=131072.0000 max=132384 max_rank=5 min=126474 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=88076 max_load_reduction_pct=0.399510 moved_to_replicas=310941 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[149505, 185927, 133195, 186025, 149840, 1792, 85559, 156733] avg=131072.0000 max=186025 max_rank=3 min=1792 min_rank=5 imbalance_ratio=1.419258 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] avg=131072.0000 max=132384 max_rank=1 min=124652 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53641 max_load_reduction_pct=0.288354 moved_to_replicas=321000 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[127322, 132384, 132383, 132381, 132383, 132385, 132382, 126956] avg=131072.0000 max=132385 max_rank=5 min=126956 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=76305 max_load_reduction_pct=0.365638 moved_to_replicas=254920 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] avg=131072.0000 max=132384 max_rank=5 min=126474 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=88076 max_load_reduction_pct=0.399510 moved_to_replicas=310941 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[91161, 91795, 220460, 41229, 159228, 139301, 170967, 134435] avg=131072.0000 max=220460 max_rank=2 min=41229 min_rank=3 imbalance_ratio=1.681976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] avg=131072.0000 max=132384 max_rank=1 min=124652 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53641 max_load_reduction_pct=0.288354 moved_to_replicas=321000 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[149505, 185927, 133195, 186025, 149840, 1792, 85559, 156733] avg=131072.0000 max=186025 max_rank=3 min=1792 min_rank=5 imbalance_ratio=1.419258 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[91161, 91795, 220460, 41229, 159228, 139301, 170967, 134435] avg=131072.0000 max=220460 max_rank=2 min=41229 min_rank=3 imbalance_ratio=1.681976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[149505, 185927, 133195, 186025, 149840, 1792, 85559, 156733] avg=131072.0000 max=186025 max_rank=3 min=1792 min_rank=5 imbalance_ratio=1.419258 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] avg=131072.0000 max=132384 max_rank=5 min=126474 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=88076 max_load_reduction_pct=0.399510 moved_to_replicas=310941 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] avg=131072.0000 max=132384 max_rank=1 min=124652 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53641 max_load_reduction_pct=0.288354 moved_to_replicas=321000 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[145615, 134762, 46623, 130011, 138658, 162308, 169523, 121076] avg=131072.0000 max=169523 max_rank=6 min=46623 min_rank=2 imbalance_ratio=1.293358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[91161, 91795, 220460, 41229, 159228, 139301, 170967, 134435] avg=131072.0000 max=220460 max_rank=2 min=41229 min_rank=3 imbalance_ratio=1.681976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] avg=131072.0000 max=132384 max_rank=1 min=124652 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53641 max_load_reduction_pct=0.288354 moved_to_replicas=321000 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[149505, 185927, 133195, 186025, 149840, 1792, 85559, 156733] avg=131072.0000 max=186025 max_rank=3 min=1792 min_rank=5 imbalance_ratio=1.419258 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] avg=131072.0000 max=132384 max_rank=5 min=126474 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=88076 max_load_reduction_pct=0.399510 moved_to_replicas=310941 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[145615, 134762, 46623, 130011, 138658, 162308, 169523, 121076] avg=131072.0000 max=169523 max_rank=6 min=46623 min_rank=2 imbalance_ratio=1.293358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] avg=131072.0000 max=132385 max_rank=4 min=127349 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=37138 max_load_reduction_pct=0.219074 moved_to_replicas=194826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[127803, 126474, 132382, 132383, 132383, 132384, 132384, 132383] avg=131072.0000 max=132384 max_rank=5 min=126474 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=88076 max_load_reduction_pct=0.399510 moved_to_replicas=310941 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[149505, 185927, 133195, 186025, 149840, 1792, 85559, 156733] avg=131072.0000 max=186025 max_rank=3 min=1792 min_rank=5 imbalance_ratio=1.419258 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] avg=131072.0000 max=132384 max_rank=1 min=124652 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53641 max_load_reduction_pct=0.288354 moved_to_replicas=321000 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[145615, 134762, 46623, 130011, 138658, 162308, 169523, 121076] avg=131072.0000 max=169523 max_rank=6 min=46623 min_rank=2 imbalance_ratio=1.293358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] avg=131072.0000 max=132385 max_rank=4 min=127349 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=37138 max_load_reduction_pct=0.219074 moved_to_replicas=194826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[145615, 134762, 46623, 130011, 138658, 162308, 169523, 121076] avg=131072.0000 max=169523 max_rank=6 min=46623 min_rank=2 imbalance_ratio=1.293358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[149505, 185927, 133195, 186025, 149840, 1792, 85559, 156733] avg=131072.0000 max=186025 max_rank=3 min=1792 min_rank=5 imbalance_ratio=1.419258 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] avg=131072.0000 max=132384 max_rank=1 min=124652 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53641 max_load_reduction_pct=0.288354 moved_to_replicas=321000 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[102783, 130537, 195021, 97331, 177188, 71545, 96437, 177734] avg=131072.0000 max=195021 max_rank=2 min=71545 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] avg=131072.0000 max=132385 max_rank=4 min=127349 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=37138 max_load_reduction_pct=0.219074 moved_to_replicas=194826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[149505, 185927, 133195, 186025, 149840, 1792, 85559, 156733] avg=131072.0000 max=186025 max_rank=3 min=1792 min_rank=5 imbalance_ratio=1.419258 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] avg=131072.0000 max=132385 max_rank=4 min=127349 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=37138 max_load_reduction_pct=0.219074 moved_to_replicas=194826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[145615, 134762, 46623, 130011, 138658, 162308, 169523, 121076] avg=131072.0000 max=169523 max_rank=6 min=46623 min_rank=2 imbalance_ratio=1.293358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[102783, 130537, 195021, 97331, 177188, 71545, 96437, 177734] avg=131072.0000 max=195021 max_rank=2 min=71545 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] avg=131072.0000 max=132384 max_rank=1 min=124652 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53641 max_load_reduction_pct=0.288354 moved_to_replicas=321000 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126380 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62637 max_load_reduction_pct=0.321181 moved_to_replicas=304880 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[132383, 132384, 132383, 132383, 132384, 129625, 124652, 132382] avg=131072.0000 max=132384 max_rank=1 min=124652 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53641 max_load_reduction_pct=0.288354 moved_to_replicas=321000 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] avg=131072.0000 max=132385 max_rank=4 min=127349 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=37138 max_load_reduction_pct=0.219074 moved_to_replicas=194826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126380 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62637 max_load_reduction_pct=0.321181 moved_to_replicas=304880 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[145615, 134762, 46623, 130011, 138658, 162308, 169523, 121076] avg=131072.0000 max=169523 max_rank=6 min=46623 min_rank=2 imbalance_ratio=1.293358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[102783, 130537, 195021, 97331, 177188, 71545, 96437, 177734] avg=131072.0000 max=195021 max_rank=2 min=71545 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[102783, 130537, 195021, 97331, 177188, 71545, 96437, 177734] avg=131072.0000 max=195021 max_rank=2 min=71545 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[145615, 134762, 46623, 130011, 138658, 162308, 169523, 121076] avg=131072.0000 max=169523 max_rank=6 min=46623 min_rank=2 imbalance_ratio=1.293358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[131103, 157739, 104701, 189084, 111057, 113216, 161523, 80153] avg=131072.0000 max=189084 max_rank=3 min=80153 min_rank=7 imbalance_ratio=1.442596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[145615, 134762, 46623, 130011, 138658, 162308, 169523, 121076] avg=131072.0000 max=169523 max_rank=6 min=46623 min_rank=2 imbalance_ratio=1.293358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] avg=131072.0000 max=132385 max_rank=4 min=127349 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=37138 max_load_reduction_pct=0.219074 moved_to_replicas=194826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126380 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62637 max_load_reduction_pct=0.321181 moved_to_replicas=304880 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126380 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62637 max_load_reduction_pct=0.321181 moved_to_replicas=304880 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[102783, 130537, 195021, 97331, 177188, 71545, 96437, 177734] avg=131072.0000 max=195021 max_rank=2 min=71545 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[131103, 157739, 104701, 189084, 111057, 113216, 161523, 80153] avg=131072.0000 max=189084 max_rank=3 min=80153 min_rank=7 imbalance_ratio=1.442596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] avg=131072.0000 max=132385 max_rank=4 min=127349 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=37138 max_load_reduction_pct=0.219074 moved_to_replicas=194826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] avg=131072.0000 max=132384 max_rank=4 min=126833 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.299867 moved_to_replicas=220412 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[132383, 132383, 129300, 130011, 132385, 132383, 132382, 127349] avg=131072.0000 max=132385 max_rank=4 min=127349 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=37138 max_load_reduction_pct=0.219074 moved_to_replicas=194826 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126380 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62637 max_load_reduction_pct=0.321181 moved_to_replicas=304880 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] avg=131072.0000 max=132384 max_rank=4 min=126833 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.299867 moved_to_replicas=220412 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[131103, 157739, 104701, 189084, 111057, 113216, 161523, 80153] avg=131072.0000 max=189084 max_rank=3 min=80153 min_rank=7 imbalance_ratio=1.442596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[131103, 157739, 104701, 189084, 111057, 113216, 161523, 80153] avg=131072.0000 max=189084 max_rank=3 min=80153 min_rank=7 imbalance_ratio=1.442596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[102783, 130537, 195021, 97331, 177188, 71545, 96437, 177734] avg=131072.0000 max=195021 max_rank=2 min=71545 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[102783, 130537, 195021, 97331, 177188, 71545, 96437, 177734] avg=131072.0000 max=195021 max_rank=2 min=71545 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[88522, 132207, 111217, 276830, 76961, 136522, 137807, 88510] avg=131072.0000 max=276830 max_rank=3 min=76961 min_rank=4 imbalance_ratio=2.112045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] avg=131072.0000 max=132384 max_rank=4 min=126833 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.299867 moved_to_replicas=220412 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] avg=131072.0000 max=132384 max_rank=4 min=126833 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.299867 moved_to_replicas=220412 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[102783, 130537, 195021, 97331, 177188, 71545, 96437, 177734] avg=131072.0000 max=195021 max_rank=2 min=71545 min_rank=5 imbalance_ratio=1.487892 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126380 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62637 max_load_reduction_pct=0.321181 moved_to_replicas=304880 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[131103, 157739, 104701, 189084, 111057, 113216, 161523, 80153] avg=131072.0000 max=189084 max_rank=3 min=80153 min_rank=7 imbalance_ratio=1.442596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[88522, 132207, 111217, 276830, 76961, 136522, 137807, 88510] avg=131072.0000 max=276830 max_rank=3 min=76961 min_rank=4 imbalance_ratio=2.112045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126380 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62637 max_load_reduction_pct=0.321181 moved_to_replicas=304880 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] avg=131072.0000 max=132385 max_rank=2 min=126661 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=144445 max_load_reduction_pct=0.521782 moved_to_replicas=301126 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[88522, 132207, 111217, 276830, 76961, 136522, 137807, 88510] avg=131072.0000 max=276830 max_rank=3 min=76961 min_rank=4 imbalance_ratio=2.112045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[132059, 130537, 132383, 130068, 132383, 132382, 126380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126380 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=62637 max_load_reduction_pct=0.321181 moved_to_replicas=304880 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[88522, 132207, 111217, 276830, 76961, 136522, 137807, 88510] avg=131072.0000 max=276830 max_rank=3 min=76961 min_rank=4 imbalance_ratio=2.112045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] avg=131072.0000 max=132384 max_rank=4 min=126833 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.299867 moved_to_replicas=220412 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] avg=131072.0000 max=132385 max_rank=2 min=126661 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=144445 max_load_reduction_pct=0.521782 moved_to_replicas=301126 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[131103, 157739, 104701, 189084, 111057, 113216, 161523, 80153] avg=131072.0000 max=189084 max_rank=3 min=80153 min_rank=7 imbalance_ratio=1.442596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[131103, 157739, 104701, 189084, 111057, 113216, 161523, 80153] avg=131072.0000 max=189084 max_rank=3 min=80153 min_rank=7 imbalance_ratio=1.442596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[121429, 114994, 191056, 131329, 163504, 127666, 196334, 2264] avg=131072.0000 max=196334 max_rank=6 min=2264 min_rank=7 imbalance_ratio=1.497910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] avg=131072.0000 max=132385 max_rank=2 min=126661 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=144445 max_load_reduction_pct=0.521782 moved_to_replicas=301126 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] avg=131072.0000 max=132385 max_rank=2 min=126661 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=144445 max_load_reduction_pct=0.521782 moved_to_replicas=301126 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] avg=131072.0000 max=132384 max_rank=4 min=126833 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.299867 moved_to_replicas=220412 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[131103, 157739, 104701, 189084, 111057, 113216, 161523, 80153] avg=131072.0000 max=189084 max_rank=3 min=80153 min_rank=7 imbalance_ratio=1.442596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[88522, 132207, 111217, 276830, 76961, 136522, 137807, 88510] avg=131072.0000 max=276830 max_rank=3 min=76961 min_rank=4 imbalance_ratio=2.112045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[121429, 114994, 191056, 131329, 163504, 127666, 196334, 2264] avg=131072.0000 max=196334 max_rank=6 min=2264 min_rank=7 imbalance_ratio=1.497910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] avg=131072.0000 max=132384 max_rank=4 min=126833 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.299867 moved_to_replicas=220412 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] avg=131072.0000 max=132384 max_rank=2 min=127665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63950 max_load_reduction_pct=0.325720 moved_to_replicas=309823 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[121429, 114994, 191056, 131329, 163504, 127666, 196334, 2264] avg=131072.0000 max=196334 max_rank=6 min=2264 min_rank=7 imbalance_ratio=1.497910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[121429, 114994, 191056, 131329, 163504, 127666, 196334, 2264] avg=131072.0000 max=196334 max_rank=6 min=2264 min_rank=7 imbalance_ratio=1.497910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[131103, 132382, 128725, 132383, 132384, 132383, 132383, 126833] avg=131072.0000 max=132384 max_rank=4 min=126833 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=56700 max_load_reduction_pct=0.299867 moved_to_replicas=220412 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] avg=131072.0000 max=132385 max_rank=2 min=126661 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=144445 max_load_reduction_pct=0.521782 moved_to_replicas=301126 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] avg=131072.0000 max=132384 max_rank=2 min=127665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63950 max_load_reduction_pct=0.325720 moved_to_replicas=309823 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[88522, 132207, 111217, 276830, 76961, 136522, 137807, 88510] avg=131072.0000 max=276830 max_rank=3 min=76961 min_rank=4 imbalance_ratio=2.112045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[166732, 225640, 105830, 96949, 43180, 117081, 131680, 161484] avg=131072.0000 max=225640 max_rank=1 min=43180 min_rank=4 imbalance_ratio=1.721497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[88522, 132207, 111217, 276830, 76961, 136522, 137807, 88510] avg=131072.0000 max=276830 max_rank=3 min=76961 min_rank=4 imbalance_ratio=2.112045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] avg=131072.0000 max=132384 max_rank=2 min=127665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63950 max_load_reduction_pct=0.325720 moved_to_replicas=309823 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] avg=131072.0000 max=132384 max_rank=2 min=127665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63950 max_load_reduction_pct=0.325720 moved_to_replicas=309823 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] avg=131072.0000 max=132385 max_rank=2 min=126661 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=144445 max_load_reduction_pct=0.521782 moved_to_replicas=301126 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] avg=131072.0000 max=132385 max_rank=4 min=124752 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=93255 max_load_reduction_pct=0.413291 moved_to_replicas=314864 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[88522, 132207, 111217, 276830, 76961, 136522, 137807, 88510] avg=131072.0000 max=276830 max_rank=3 min=76961 min_rank=4 imbalance_ratio=2.112045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[121429, 114994, 191056, 131329, 163504, 127666, 196334, 2264] avg=131072.0000 max=196334 max_rank=6 min=2264 min_rank=7 imbalance_ratio=1.497910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] avg=131072.0000 max=132385 max_rank=2 min=126661 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=144445 max_load_reduction_pct=0.521782 moved_to_replicas=301126 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[166732, 225640, 105830, 96949, 43180, 117081, 131680, 161484] avg=131072.0000 max=225640 max_rank=1 min=43180 min_rank=4 imbalance_ratio=1.721497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[166732, 225640, 105830, 96949, 43180, 117081, 131680, 161484] avg=131072.0000 max=225640 max_rank=1 min=43180 min_rank=4 imbalance_ratio=1.721497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[166732, 225640, 105830, 96949, 43180, 117081, 131680, 161484] avg=131072.0000 max=225640 max_rank=1 min=43180 min_rank=4 imbalance_ratio=1.721497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[127791, 132207, 132385, 132382, 132382, 132384, 132384, 126661] avg=131072.0000 max=132385 max_rank=2 min=126661 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=144445 max_load_reduction_pct=0.521782 moved_to_replicas=301126 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] avg=131072.0000 max=132384 max_rank=2 min=127665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63950 max_load_reduction_pct=0.325720 moved_to_replicas=309823 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[121429, 114994, 191056, 131329, 163504, 127666, 196334, 2264] avg=131072.0000 max=196334 max_rank=6 min=2264 min_rank=7 imbalance_ratio=1.497910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] avg=131072.0000 max=132385 max_rank=4 min=124752 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=93255 max_load_reduction_pct=0.413291 moved_to_replicas=314864 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[134704, 141787, 130650, 129477, 130506, 129564, 84329, 167559] avg=131072.0000 max=167559 max_rank=7 min=84329 min_rank=6 imbalance_ratio=1.278374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] avg=131072.0000 max=132385 max_rank=4 min=124752 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=93255 max_load_reduction_pct=0.413291 moved_to_replicas=314864 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] avg=131072.0000 max=132385 max_rank=4 min=124752 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=93255 max_load_reduction_pct=0.413291 moved_to_replicas=314864 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[121429, 114994, 191056, 131329, 163504, 127666, 196334, 2264] avg=131072.0000 max=196334 max_rank=6 min=2264 min_rank=7 imbalance_ratio=1.497910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] avg=131072.0000 max=132384 max_rank=2 min=127665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63950 max_load_reduction_pct=0.325720 moved_to_replicas=309823 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] avg=131072.0000 max=132384 max_rank=1 min=129477 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35175 max_load_reduction_pct=0.209926 moved_to_replicas=96205 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[121429, 114994, 191056, 131329, 163504, 127666, 196334, 2264] avg=131072.0000 max=196334 max_rank=6 min=2264 min_rank=7 imbalance_ratio=1.497910 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[166732, 225640, 105830, 96949, 43180, 117081, 131680, 161484] avg=131072.0000 max=225640 max_rank=1 min=43180 min_rank=4 imbalance_ratio=1.721497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[134704, 141787, 130650, 129477, 130506, 129564, 84329, 167559] avg=131072.0000 max=167559 max_rank=7 min=84329 min_rank=6 imbalance_ratio=1.278374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] avg=131072.0000 max=132384 max_rank=2 min=127665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63950 max_load_reduction_pct=0.325720 moved_to_replicas=309823 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[134704, 141787, 130650, 129477, 130506, 129564, 84329, 167559] avg=131072.0000 max=167559 max_rank=7 min=84329 min_rank=6 imbalance_ratio=1.278374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[134704, 141787, 130650, 129477, 130506, 129564, 84329, 167559] avg=131072.0000 max=167559 max_rank=7 min=84329 min_rank=6 imbalance_ratio=1.278374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[132383, 132383, 132384, 131329, 132383, 127666, 132383, 127665] avg=131072.0000 max=132384 max_rank=2 min=127665 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=63950 max_load_reduction_pct=0.325720 moved_to_replicas=309823 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] avg=131072.0000 max=132385 max_rank=4 min=124752 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=93255 max_load_reduction_pct=0.413291 moved_to_replicas=314864 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] avg=131072.0000 max=132384 max_rank=1 min=129477 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35175 max_load_reduction_pct=0.209926 moved_to_replicas=96205 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] avg=131072.0000 max=132384 max_rank=1 min=129477 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35175 max_load_reduction_pct=0.209926 moved_to_replicas=96205 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[122107, 157910, 100308, 124109, 153148, 104294, 99214, 187486] avg=131072.0000 max=187486 max_rank=7 min=99214 min_rank=6 imbalance_ratio=1.430405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[166732, 225640, 105830, 96949, 43180, 117081, 131680, 161484] avg=131072.0000 max=225640 max_rank=1 min=43180 min_rank=4 imbalance_ratio=1.721497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] avg=131072.0000 max=132384 max_rank=1 min=129477 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35175 max_load_reduction_pct=0.209926 moved_to_replicas=96205 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[166732, 225640, 105830, 96949, 43180, 117081, 131680, 161484] avg=131072.0000 max=225640 max_rank=1 min=43180 min_rank=4 imbalance_ratio=1.721497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] avg=131072.0000 max=132384 max_rank=7 min=124891 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55102 max_load_reduction_pct=0.293899 moved_to_replicas=203692 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[166732, 225640, 105830, 96949, 43180, 117081, 131680, 161484] avg=131072.0000 max=225640 max_rank=1 min=43180 min_rank=4 imbalance_ratio=1.721497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[122107, 157910, 100308, 124109, 153148, 104294, 99214, 187486] avg=131072.0000 max=187486 max_rank=7 min=99214 min_rank=6 imbalance_ratio=1.430405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[134704, 141787, 130650, 129477, 130506, 129564, 84329, 167559] avg=131072.0000 max=167559 max_rank=7 min=84329 min_rank=6 imbalance_ratio=1.278374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] avg=131072.0000 max=132385 max_rank=4 min=124752 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=93255 max_load_reduction_pct=0.413291 moved_to_replicas=314864 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] avg=131072.0000 max=132385 max_rank=4 min=124752 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=93255 max_load_reduction_pct=0.413291 moved_to_replicas=314864 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[122107, 157910, 100308, 124109, 153148, 104294, 99214, 187486] avg=131072.0000 max=187486 max_rank=7 min=99214 min_rank=6 imbalance_ratio=1.430405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[122107, 157910, 100308, 124109, 153148, 104294, 99214, 187486] avg=131072.0000 max=187486 max_rank=7 min=99214 min_rank=6 imbalance_ratio=1.430405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[132383, 132384, 132381, 124752, 132385, 130228, 131680, 132383] avg=131072.0000 max=132385 max_rank=4 min=124752 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=93255 max_load_reduction_pct=0.413291 moved_to_replicas=314864 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] avg=131072.0000 max=132384 max_rank=7 min=124891 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55102 max_load_reduction_pct=0.293899 moved_to_replicas=203692 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] avg=131072.0000 max=132384 max_rank=1 min=129477 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35175 max_load_reduction_pct=0.209926 moved_to_replicas=96205 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] avg=131072.0000 max=132384 max_rank=7 min=124891 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55102 max_load_reduction_pct=0.293899 moved_to_replicas=203692 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[164130, 141800, 124159, 127429, 165931, 137625, 98954, 88548] avg=131072.0000 max=165931 max_rank=4 min=88548 min_rank=7 imbalance_ratio=1.265953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] avg=131072.0000 max=132384 max_rank=7 min=124891 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55102 max_load_reduction_pct=0.293899 moved_to_replicas=203692 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[134704, 141787, 130650, 129477, 130506, 129564, 84329, 167559] avg=131072.0000 max=167559 max_rank=7 min=84329 min_rank=6 imbalance_ratio=1.278374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[134704, 141787, 130650, 129477, 130506, 129564, 84329, 167559] avg=131072.0000 max=167559 max_rank=7 min=84329 min_rank=6 imbalance_ratio=1.278374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] avg=131072.0000 max=132385 max_rank=1 min=127429 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=33546 max_load_reduction_pct=0.202168 moved_to_replicas=150168 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[134704, 141787, 130650, 129477, 130506, 129564, 84329, 167559] avg=131072.0000 max=167559 max_rank=7 min=84329 min_rank=6 imbalance_ratio=1.278374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[164130, 141800, 124159, 127429, 165931, 137625, 98954, 88548] avg=131072.0000 max=165931 max_rank=4 min=88548 min_rank=7 imbalance_ratio=1.265953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[164130, 141800, 124159, 127429, 165931, 137625, 98954, 88548] avg=131072.0000 max=165931 max_rank=4 min=88548 min_rank=7 imbalance_ratio=1.265953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] avg=131072.0000 max=132384 max_rank=1 min=129477 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35175 max_load_reduction_pct=0.209926 moved_to_replicas=96205 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] avg=131072.0000 max=132384 max_rank=1 min=129477 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35175 max_load_reduction_pct=0.209926 moved_to_replicas=96205 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[122107, 157910, 100308, 124109, 153148, 104294, 99214, 187486] avg=131072.0000 max=187486 max_rank=7 min=99214 min_rank=6 imbalance_ratio=1.430405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[164130, 141800, 124159, 127429, 165931, 137625, 98954, 88548] avg=131072.0000 max=165931 max_rank=4 min=88548 min_rank=7 imbalance_ratio=1.265953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[132383, 132384, 130650, 129477, 130506, 129564, 131229, 132383] avg=131072.0000 max=132384 max_rank=1 min=129477 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35175 max_load_reduction_pct=0.209926 moved_to_replicas=96205 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] avg=131072.0000 max=132385 max_rank=1 min=127429 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=33546 max_load_reduction_pct=0.202168 moved_to_replicas=150168 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[148624, 63482, 175853, 228875, 166932, 83232, 114418, 67160] avg=131072.0000 max=228875 max_rank=3 min=63482 min_rank=1 imbalance_ratio=1.746178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] avg=131072.0000 max=132385 max_rank=1 min=127429 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=33546 max_load_reduction_pct=0.202168 moved_to_replicas=150168 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] avg=131072.0000 max=132384 max_rank=7 min=124891 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55102 max_load_reduction_pct=0.293899 moved_to_replicas=203692 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] avg=131072.0000 max=132385 max_rank=1 min=127429 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=33546 max_load_reduction_pct=0.202168 moved_to_replicas=150168 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[122107, 157910, 100308, 124109, 153148, 104294, 99214, 187486] avg=131072.0000 max=187486 max_rank=7 min=99214 min_rank=6 imbalance_ratio=1.430405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[122107, 157910, 100308, 124109, 153148, 104294, 99214, 187486] avg=131072.0000 max=187486 max_rank=7 min=99214 min_rank=6 imbalance_ratio=1.430405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125349 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96491 max_load_reduction_pct=0.421588 moved_to_replicas=369521 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[148624, 63482, 175853, 228875, 166932, 83232, 114418, 67160] avg=131072.0000 max=228875 max_rank=3 min=63482 min_rank=1 imbalance_ratio=1.746178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[122107, 157910, 100308, 124109, 153148, 104294, 99214, 187486] avg=131072.0000 max=187486 max_rank=7 min=99214 min_rank=6 imbalance_ratio=1.430405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[148624, 63482, 175853, 228875, 166932, 83232, 114418, 67160] avg=131072.0000 max=228875 max_rank=3 min=63482 min_rank=1 imbalance_ratio=1.746178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] avg=131072.0000 max=132384 max_rank=7 min=124891 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55102 max_load_reduction_pct=0.293899 moved_to_replicas=203692 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[148624, 63482, 175853, 228875, 166932, 83232, 114418, 67160] avg=131072.0000 max=228875 max_rank=3 min=63482 min_rank=1 imbalance_ratio=1.746178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[164130, 141800, 124159, 127429, 165931, 137625, 98954, 88548] avg=131072.0000 max=165931 max_rank=4 min=88548 min_rank=7 imbalance_ratio=1.265953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] avg=131072.0000 max=132384 max_rank=7 min=124891 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55102 max_load_reduction_pct=0.293899 moved_to_replicas=203692 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125349 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96491 max_load_reduction_pct=0.421588 moved_to_replicas=369521 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[132383, 132381, 132382, 124891, 132383, 129823, 131949, 132384] avg=131072.0000 max=132384 max_rank=7 min=124891 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55102 max_load_reduction_pct=0.293899 moved_to_replicas=203692 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125349 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96491 max_load_reduction_pct=0.421588 moved_to_replicas=369521 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[144006, 53341, 138977, 150885, 174213, 98809, 65397, 222948] avg=131072.0000 max=222948 max_rank=7 min=53341 min_rank=1 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125349 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96491 max_load_reduction_pct=0.421588 moved_to_replicas=369521 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] avg=131072.0000 max=132385 max_rank=1 min=127429 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=33546 max_load_reduction_pct=0.202168 moved_to_replicas=150168 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[164130, 141800, 124159, 127429, 165931, 137625, 98954, 88548] avg=131072.0000 max=165931 max_rank=4 min=88548 min_rank=7 imbalance_ratio=1.265953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] avg=131072.0000 max=132383 max_rank=3 min=126435 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90565 max_load_reduction_pct=0.406216 moved_to_replicas=342757 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[164130, 141800, 124159, 127429, 165931, 137625, 98954, 88548] avg=131072.0000 max=165931 max_rank=4 min=88548 min_rank=7 imbalance_ratio=1.265953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[144006, 53341, 138977, 150885, 174213, 98809, 65397, 222948] avg=131072.0000 max=222948 max_rank=7 min=53341 min_rank=1 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[144006, 53341, 138977, 150885, 174213, 98809, 65397, 222948] avg=131072.0000 max=222948 max_rank=7 min=53341 min_rank=1 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[164130, 141800, 124159, 127429, 165931, 137625, 98954, 88548] avg=131072.0000 max=165931 max_rank=4 min=88548 min_rank=7 imbalance_ratio=1.265953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[144006, 53341, 138977, 150885, 174213, 98809, 65397, 222948] avg=131072.0000 max=222948 max_rank=7 min=53341 min_rank=1 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] avg=131072.0000 max=132385 max_rank=1 min=127429 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=33546 max_load_reduction_pct=0.202168 moved_to_replicas=150168 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[148624, 63482, 175853, 228875, 166932, 83232, 114418, 67160] avg=131072.0000 max=228875 max_rank=3 min=63482 min_rank=1 imbalance_ratio=1.746178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] avg=131072.0000 max=132385 max_rank=1 min=127429 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=33546 max_load_reduction_pct=0.202168 moved_to_replicas=150168 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] avg=131072.0000 max=132383 max_rank=3 min=126435 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90565 max_load_reduction_pct=0.406216 moved_to_replicas=342757 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] avg=131072.0000 max=132383 max_rank=3 min=126435 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90565 max_load_reduction_pct=0.406216 moved_to_replicas=342757 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[132383, 132385, 129400, 127429, 132383, 132384, 131531, 130681] avg=131072.0000 max=132385 max_rank=1 min=127429 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=33546 max_load_reduction_pct=0.202168 moved_to_replicas=150168 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] avg=131072.0000 max=132383 max_rank=3 min=126435 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90565 max_load_reduction_pct=0.406216 moved_to_replicas=342757 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125349 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96491 max_load_reduction_pct=0.421588 moved_to_replicas=369521 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[148624, 63482, 175853, 228875, 166932, 83232, 114418, 67160] avg=131072.0000 max=228875 max_rank=3 min=63482 min_rank=1 imbalance_ratio=1.746178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[148624, 63482, 175853, 228875, 166932, 83232, 114418, 67160] avg=131072.0000 max=228875 max_rank=3 min=63482 min_rank=1 imbalance_ratio=1.746178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[148624, 63482, 175853, 228875, 166932, 83232, 114418, 67160] avg=131072.0000 max=228875 max_rank=3 min=63482 min_rank=1 imbalance_ratio=1.746178 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125349 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96491 max_load_reduction_pct=0.421588 moved_to_replicas=369521 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[144006, 53341, 138977, 150885, 174213, 98809, 65397, 222948] avg=131072.0000 max=222948 max_rank=7 min=53341 min_rank=1 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125349 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96491 max_load_reduction_pct=0.421588 moved_to_replicas=369521 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[132384, 128930, 132383, 132383, 132383, 125349, 132382, 132382] avg=131072.0000 max=132384 max_rank=0 min=125349 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=96491 max_load_reduction_pct=0.421588 moved_to_replicas=369521 active_clone_links=19 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] avg=131072.0000 max=132383 max_rank=3 min=126435 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90565 max_load_reduction_pct=0.406216 moved_to_replicas=342757 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[144006, 53341, 138977, 150885, 174213, 98809, 65397, 222948] avg=131072.0000 max=222948 max_rank=7 min=53341 min_rank=1 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[144006, 53341, 138977, 150885, 174213, 98809, 65397, 222948] avg=131072.0000 max=222948 max_rank=7 min=53341 min_rank=1 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[144006, 53341, 138977, 150885, 174213, 98809, 65397, 222948] avg=131072.0000 max=222948 max_rank=7 min=53341 min_rank=1 imbalance_ratio=1.700958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] avg=131072.0000 max=132383 max_rank=3 min=126435 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90565 max_load_reduction_pct=0.406216 moved_to_replicas=342757 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] avg=131072.0000 max=132383 max_rank=3 min=126435 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90565 max_load_reduction_pct=0.406216 moved_to_replicas=342757 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[132382, 129403, 132382, 132383, 132383, 126435, 130825, 132383] avg=131072.0000 max=132383 max_rank=3 min=126435 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90565 max_load_reduction_pct=0.406216 moved_to_replicas=342757 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=4201577 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1000192 peak_demand=134408 required_capacity=1.025452 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=4183208 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017216 peak_demand=132722 required_capacity=1.012589 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=4167293 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1033216 peak_demand=134407 required_capacity=1.025444 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=4183454 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015424 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=4209982 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=992128 peak_demand=134407 required_capacity=1.025444 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=4184087 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015680 peak_demand=134408 required_capacity=1.025452 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=4232958 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=967936 peak_demand=134408 required_capacity=1.025452 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=4191873 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1006592 peak_demand=132384 required_capacity=1.010010 + [2026-08-30 05:17:53] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 580.3 | throughput per GPU (TFLOP/s/GPU): 574.5 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.301675E+01 | loss scale: 1.0 | grad norm: 46.012 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=163840 expected_recv=259880 expected_overflow=0 exact_recv_per_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=163840 expected_recv=260208 expected_overflow=0 exact_recv_per_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=163840 expected_recv=253234 expected_overflow=0 exact_recv_per_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=163840 expected_recv=259880 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=163840 expected_recv=260208 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=163840 expected_recv=253234 actual_valid_recv=255616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=255616 slack_slots=72064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=163840 expected_recv=246953 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=163840 expected_recv=246953 actual_valid_recv=249984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262397 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=163840 expected_recv=263972 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=249984 slack_slots=77696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262397 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267904 slack_slots=59776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=163840 expected_recv=263972 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=265600 slack_slots=62080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=163840 expected_recv=256981 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=163840 expected_recv=254083 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=163840 expected_recv=256981 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=163840 expected_recv=254083 actual_valid_recv=256896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=260352 slack_slots=67328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=256896 slack_slots=70784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262258 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267904 slack_slots=59776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262258 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=163840 expected_recv=261249 expected_overflow=0 exact_recv_per_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262182 expected_overflow=0 exact_recv_per_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=163840 expected_recv=261249 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262182 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=163840 expected_recv=253107 expected_overflow=0 exact_recv_per_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=163840 expected_recv=261552 expected_overflow=0 exact_recv_per_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=163840 expected_recv=253107 actual_valid_recv=255872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=163840 expected_recv=261552 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=255872 slack_slots=71808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=163840 expected_recv=258261 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=163840 expected_recv=258298 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=163840 expected_recv=258261 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=260864 slack_slots=66816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=163840 expected_recv=258298 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=163840 expected_recv=256762 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=260608 slack_slots=67072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=163840 expected_recv=256762 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=259328 slack_slots=68352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262021 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=163840 expected_recv=260880 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262021 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=163840 expected_recv=260880 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=163840 expected_recv=254441 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=163840 expected_recv=260746 expected_overflow=0 exact_recv_per_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=163840 expected_recv=254441 actual_valid_recv=256640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=163840 expected_recv=260746 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=256640 slack_slots=71040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=163840 expected_recv=261158 expected_overflow=0 exact_recv_per_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=163840 expected_recv=261158 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=163840 expected_recv=255868 expected_overflow=0 exact_recv_per_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=163840 expected_recv=255868 actual_valid_recv=258560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=163840 expected_recv=261351 expected_overflow=0 exact_recv_per_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=258560 slack_slots=69120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=163840 expected_recv=261351 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=163840 expected_recv=259714 expected_overflow=0 exact_recv_per_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=163840 expected_recv=259714 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=163840 expected_recv=261804 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=163840 expected_recv=261804 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262860 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=163840 expected_recv=254202 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262860 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=163840 expected_recv=264564 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=163840 expected_recv=254202 actual_valid_recv=256256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265472 slack_slots=62208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=163840 expected_recv=264564 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=256256 slack_slots=71424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=163840 expected_recv=259424 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=163840 expected_recv=259424 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=163840 expected_recv=263489 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=163840 expected_recv=260197 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=163840 expected_recv=263489 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=163840 expected_recv=260197 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=163840 expected_recv=262215 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=163840 expected_recv=258649 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=163840 expected_recv=259572 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=163840 expected_recv=262215 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=163840 expected_recv=258649 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=163840 expected_recv=259572 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=163840 expected_recv=263498 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=163840 expected_recv=263498 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=163840 expected_recv=256602 expected_overflow=0 exact_recv_per_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=163840 expected_recv=256602 actual_valid_recv=258688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262718 expected_overflow=0 exact_recv_per_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=258688 slack_slots=68992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=163840 expected_recv=256358 expected_overflow=0 exact_recv_per_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262718 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262407 expected_overflow=0 exact_recv_per_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=163840 expected_recv=256358 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262407 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=259072 slack_slots=68608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=163840 expected_recv=264497 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=163840 expected_recv=263743 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=163840 expected_recv=264497 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=163840 expected_recv=263743 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=163840 expected_recv=261940 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=163840 expected_recv=261940 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=163840 expected_recv=247910 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=163840 expected_recv=260869 expected_overflow=0 exact_recv_per_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=163840 expected_recv=247910 actual_valid_recv=250496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=163840 expected_recv=260559 expected_overflow=0 exact_recv_per_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=163840 expected_recv=260869 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=163840 expected_recv=259339 expected_overflow=0 exact_recv_per_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=250496 slack_slots=77184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=163840 expected_recv=260559 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=163840 expected_recv=259339 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=163840 expected_recv=258485 expected_overflow=0 exact_recv_per_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=163840 expected_recv=258485 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=260480 slack_slots=67200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=163840 expected_recv=257872 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=163840 expected_recv=263602 expected_overflow=0 exact_recv_per_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=163840 expected_recv=257872 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=163840 expected_recv=263602 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=259968 slack_slots=67712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261387 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261387 actual_valid_recv=262656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261500 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=163840 expected_recv=259899 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=262656 slack_slots=65024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261500 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=163840 expected_recv=259899 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=163840 expected_recv=260525 expected_overflow=0 exact_recv_per_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261075 expected_overflow=0 exact_recv_per_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=163840 expected_recv=262196 expected_overflow=0 exact_recv_per_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=163840 expected_recv=260525 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=163840 expected_recv=263928 expected_overflow=0 exact_recv_per_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261075 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=163840 expected_recv=262196 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=163840 expected_recv=263928 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=163840 expected_recv=258939 expected_overflow=0 exact_recv_per_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=163840 expected_recv=258939 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=163840 expected_recv=258386 expected_overflow=0 exact_recv_per_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=261120 slack_slots=66560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=163840 expected_recv=258386 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=163840 expected_recv=259255 expected_overflow=0 exact_recv_per_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=163840 expected_recv=255974 expected_overflow=0 exact_recv_per_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=260864 slack_slots=66816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=163840 expected_recv=259255 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=163840 expected_recv=255974 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=260864 slack_slots=66816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=257920 slack_slots=69760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262341 expected_overflow=0 exact_recv_per_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=163840 expected_recv=260518 expected_overflow=0 exact_recv_per_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262341 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=163840 expected_recv=260518 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=163840 expected_recv=264686 expected_overflow=0 exact_recv_per_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=163840 expected_recv=261558 expected_overflow=0 exact_recv_per_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=163840 expected_recv=264686 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=163840 expected_recv=261558 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=163840 expected_recv=256020 expected_overflow=0 exact_recv_per_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=163840 expected_recv=256020 actual_valid_recv=258944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=163840 expected_recv=262117 expected_overflow=0 exact_recv_per_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=258944 slack_slots=68736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=163840 expected_recv=262117 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[88233, 123840, 142243, 124145, 146622, 157699, 169758, 96036] avg=131072.0000 max=169758 max_rank=6 min=88233 min_rank=0 imbalance_ratio=1.295151 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=163840 expected_recv=258473 expected_overflow=0 exact_recv_per_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[88233, 123840, 142243, 124145, 146622, 157699, 169758, 96036] avg=131072.0000 max=169758 max_rank=6 min=88233 min_rank=0 imbalance_ratio=1.295151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[88233, 123840, 142243, 124145, 146622, 157699, 169758, 96036] avg=131072.0000 max=169758 max_rank=6 min=88233 min_rank=0 imbalance_ratio=1.295151 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=163840 expected_recv=258473 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[88233, 123840, 142243, 124145, 146622, 157699, 169758, 96036] avg=131072.0000 max=169758 max_rank=6 min=88233 min_rank=0 imbalance_ratio=1.295151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] avg=131072.0000 max=132383 max_rank=0 min=125878 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=37375 max_load_reduction_pct=0.220166 moved_to_replicas=170276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=260352 slack_slots=67328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[88233, 123840, 142243, 124145, 146622, 157699, 169758, 96036] avg=131072.0000 max=169758 max_rank=6 min=88233 min_rank=0 imbalance_ratio=1.295151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] avg=131072.0000 max=132383 max_rank=0 min=125878 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=37375 max_load_reduction_pct=0.220166 moved_to_replicas=170276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[88233, 123840, 142243, 124145, 146622, 157699, 169758, 96036] avg=131072.0000 max=169758 max_rank=6 min=88233 min_rank=0 imbalance_ratio=1.295151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[88233, 123840, 142243, 124145, 146622, 157699, 169758, 96036] avg=131072.0000 max=169758 max_rank=6 min=88233 min_rank=0 imbalance_ratio=1.295151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] avg=131072.0000 max=132383 max_rank=0 min=125878 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=37375 max_load_reduction_pct=0.220166 moved_to_replicas=170276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] avg=131072.0000 max=132383 max_rank=0 min=125878 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=37375 max_load_reduction_pct=0.220166 moved_to_replicas=170276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] avg=131072.0000 max=132383 max_rank=0 min=125878 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=37375 max_load_reduction_pct=0.220166 moved_to_replicas=170276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[148940, 72359, 66938, 133545, 169524, 164530, 98389, 194351] avg=131072.0000 max=194351 max_rank=7 min=66938 min_rank=2 imbalance_ratio=1.482780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] avg=131072.0000 max=132383 max_rank=0 min=125878 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=37375 max_load_reduction_pct=0.220166 moved_to_replicas=170276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[88233, 123840, 142243, 124145, 146622, 157699, 169758, 96036] avg=131072.0000 max=169758 max_rank=6 min=88233 min_rank=0 imbalance_ratio=1.295151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] avg=131072.0000 max=132383 max_rank=0 min=125878 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=37375 max_load_reduction_pct=0.220166 moved_to_replicas=170276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[148940, 72359, 66938, 133545, 169524, 164530, 98389, 194351] avg=131072.0000 max=194351 max_rank=7 min=66938 min_rank=2 imbalance_ratio=1.482780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[148940, 72359, 66938, 133545, 169524, 164530, 98389, 194351] avg=131072.0000 max=194351 max_rank=7 min=66938 min_rank=2 imbalance_ratio=1.482780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[148940, 72359, 66938, 133545, 169524, 164530, 98389, 194351] avg=131072.0000 max=194351 max_rank=7 min=66938 min_rank=2 imbalance_ratio=1.482780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] avg=131072.0000 max=132384 max_rank=4 min=123240 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61967 max_load_reduction_pct=0.318841 moved_to_replicas=299326 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] avg=131072.0000 max=132384 max_rank=4 min=123240 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61967 max_load_reduction_pct=0.318841 moved_to_replicas=299326 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[148940, 72359, 66938, 133545, 169524, 164530, 98389, 194351] avg=131072.0000 max=194351 max_rank=7 min=66938 min_rank=2 imbalance_ratio=1.482780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[132383, 125878, 132383, 132383, 132383, 132383, 132383, 128400] avg=131072.0000 max=132383 max_rank=0 min=125878 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=37375 max_load_reduction_pct=0.220166 moved_to_replicas=170276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] avg=131072.0000 max=132384 max_rank=4 min=123240 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61967 max_load_reduction_pct=0.318841 moved_to_replicas=299326 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] avg=131072.0000 max=132384 max_rank=4 min=123240 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61967 max_load_reduction_pct=0.318841 moved_to_replicas=299326 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[148940, 72359, 66938, 133545, 169524, 164530, 98389, 194351] avg=131072.0000 max=194351 max_rank=7 min=66938 min_rank=2 imbalance_ratio=1.482780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[148940, 72359, 66938, 133545, 169524, 164530, 98389, 194351] avg=131072.0000 max=194351 max_rank=7 min=66938 min_rank=2 imbalance_ratio=1.482780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[165756, 206884, 98865, 109599, 98396, 125701, 168669, 74706] avg=131072.0000 max=206884 max_rank=1 min=74706 min_rank=7 imbalance_ratio=1.578400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] avg=131072.0000 max=132384 max_rank=4 min=123240 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61967 max_load_reduction_pct=0.318841 moved_to_replicas=299326 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] avg=131072.0000 max=132384 max_rank=4 min=123240 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61967 max_load_reduction_pct=0.318841 moved_to_replicas=299326 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[165756, 206884, 98865, 109599, 98396, 125701, 168669, 74706] avg=131072.0000 max=206884 max_rank=1 min=74706 min_rank=7 imbalance_ratio=1.578400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] avg=131072.0000 max=132384 max_rank=4 min=123240 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61967 max_load_reduction_pct=0.318841 moved_to_replicas=299326 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[148940, 72359, 66938, 133545, 169524, 164530, 98389, 194351] avg=131072.0000 max=194351 max_rank=7 min=66938 min_rank=2 imbalance_ratio=1.482780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[165756, 206884, 98865, 109599, 98396, 125701, 168669, 74706] avg=131072.0000 max=206884 max_rank=1 min=74706 min_rank=7 imbalance_ratio=1.578400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=126309 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=74501 max_load_reduction_pct=0.360110 moved_to_replicas=290483 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[165756, 206884, 98865, 109599, 98396, 125701, 168669, 74706] avg=131072.0000 max=206884 max_rank=1 min=74706 min_rank=7 imbalance_ratio=1.578400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=126309 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=74501 max_load_reduction_pct=0.360110 moved_to_replicas=290483 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[165756, 206884, 98865, 109599, 98396, 125701, 168669, 74706] avg=131072.0000 max=206884 max_rank=1 min=74706 min_rank=7 imbalance_ratio=1.578400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[132382, 132383, 123240, 132383, 132384, 132383, 131038, 132383] avg=131072.0000 max=132384 max_rank=4 min=123240 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61967 max_load_reduction_pct=0.318841 moved_to_replicas=299326 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=126309 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=74501 max_load_reduction_pct=0.360110 moved_to_replicas=290483 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[165756, 206884, 98865, 109599, 98396, 125701, 168669, 74706] avg=131072.0000 max=206884 max_rank=1 min=74706 min_rank=7 imbalance_ratio=1.578400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=126309 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=74501 max_load_reduction_pct=0.360110 moved_to_replicas=290483 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[165756, 206884, 98865, 109599, 98396, 125701, 168669, 74706] avg=131072.0000 max=206884 max_rank=1 min=74706 min_rank=7 imbalance_ratio=1.578400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[77575, 130747, 146773, 66065, 142830, 165187, 192609, 126790] avg=131072.0000 max=192609 max_rank=6 min=66065 min_rank=3 imbalance_ratio=1.469490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=126309 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=74501 max_load_reduction_pct=0.360110 moved_to_replicas=290483 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[77575, 130747, 146773, 66065, 142830, 165187, 192609, 126790] avg=131072.0000 max=192609 max_rank=6 min=66065 min_rank=3 imbalance_ratio=1.469490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=126309 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=74501 max_load_reduction_pct=0.360110 moved_to_replicas=290483 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=126309 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=74501 max_load_reduction_pct=0.360110 moved_to_replicas=290483 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] avg=131072.0000 max=132385 max_rank=6 min=126790 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60224 max_load_reduction_pct=0.312675 moved_to_replicas=234463 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[77575, 130747, 146773, 66065, 142830, 165187, 192609, 126790] avg=131072.0000 max=192609 max_rank=6 min=66065 min_rank=3 imbalance_ratio=1.469490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[165756, 206884, 98865, 109599, 98396, 125701, 168669, 74706] avg=131072.0000 max=206884 max_rank=1 min=74706 min_rank=7 imbalance_ratio=1.578400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[77575, 130747, 146773, 66065, 142830, 165187, 192609, 126790] avg=131072.0000 max=192609 max_rank=6 min=66065 min_rank=3 imbalance_ratio=1.469490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] avg=131072.0000 max=132385 max_rank=6 min=126790 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60224 max_load_reduction_pct=0.312675 moved_to_replicas=234463 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[77575, 130747, 146773, 66065, 142830, 165187, 192609, 126790] avg=131072.0000 max=192609 max_rank=6 min=66065 min_rank=3 imbalance_ratio=1.469490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] avg=131072.0000 max=132385 max_rank=6 min=126790 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60224 max_load_reduction_pct=0.312675 moved_to_replicas=234463 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[77575, 130747, 146773, 66065, 142830, 165187, 192609, 126790] avg=131072.0000 max=192609 max_rank=6 min=66065 min_rank=3 imbalance_ratio=1.469490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] avg=131072.0000 max=132385 max_rank=6 min=126790 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60224 max_load_reduction_pct=0.312675 moved_to_replicas=234463 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[132383, 132383, 129213, 132383, 131140, 126309, 132383, 132382] avg=131072.0000 max=132383 max_rank=0 min=126309 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=74501 max_load_reduction_pct=0.360110 moved_to_replicas=290483 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[77575, 130747, 146773, 66065, 142830, 165187, 192609, 126790] avg=131072.0000 max=192609 max_rank=6 min=66065 min_rank=3 imbalance_ratio=1.469490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[163221, 181689, 98306, 119751, 162662, 97931, 89560, 135456] avg=131072.0000 max=181689 max_rank=1 min=89560 min_rank=6 imbalance_ratio=1.386177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] avg=131072.0000 max=132385 max_rank=6 min=126790 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60224 max_load_reduction_pct=0.312675 moved_to_replicas=234463 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[163221, 181689, 98306, 119751, 162662, 97931, 89560, 135456] avg=131072.0000 max=181689 max_rank=1 min=89560 min_rank=6 imbalance_ratio=1.386177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] avg=131072.0000 max=132385 max_rank=6 min=126790 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60224 max_load_reduction_pct=0.312675 moved_to_replicas=234463 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[163221, 181689, 98306, 119751, 162662, 97931, 89560, 135456] avg=131072.0000 max=181689 max_rank=1 min=89560 min_rank=6 imbalance_ratio=1.386177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] avg=131072.0000 max=132385 max_rank=6 min=126790 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60224 max_load_reduction_pct=0.312675 moved_to_replicas=234463 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] avg=131072.0000 max=132384 max_rank=1 min=125399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=49305 max_load_reduction_pct=0.271370 moved_to_replicas=228800 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[163221, 181689, 98306, 119751, 162662, 97931, 89560, 135456] avg=131072.0000 max=181689 max_rank=1 min=89560 min_rank=6 imbalance_ratio=1.386177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[77575, 130747, 146773, 66065, 142830, 165187, 192609, 126790] avg=131072.0000 max=192609 max_rank=6 min=66065 min_rank=3 imbalance_ratio=1.469490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] avg=131072.0000 max=132384 max_rank=1 min=125399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=49305 max_load_reduction_pct=0.271370 moved_to_replicas=228800 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] avg=131072.0000 max=132384 max_rank=1 min=125399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=49305 max_load_reduction_pct=0.271370 moved_to_replicas=228800 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[163221, 181689, 98306, 119751, 162662, 97931, 89560, 135456] avg=131072.0000 max=181689 max_rank=1 min=89560 min_rank=6 imbalance_ratio=1.386177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[163221, 181689, 98306, 119751, 162662, 97931, 89560, 135456] avg=131072.0000 max=181689 max_rank=1 min=89560 min_rank=6 imbalance_ratio=1.386177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] avg=131072.0000 max=132384 max_rank=1 min=125399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=49305 max_load_reduction_pct=0.271370 moved_to_replicas=228800 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[129921, 130747, 132382, 131584, 132384, 132383, 132385, 126790] avg=131072.0000 max=132385 max_rank=6 min=126790 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=60224 max_load_reduction_pct=0.312675 moved_to_replicas=234463 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[180872, 229277, 98120, 88695, 41495, 117555, 168501, 124061] avg=131072.0000 max=229277 max_rank=1 min=41495 min_rank=4 imbalance_ratio=1.749245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[163221, 181689, 98306, 119751, 162662, 97931, 89560, 135456] avg=131072.0000 max=181689 max_rank=1 min=89560 min_rank=6 imbalance_ratio=1.386177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] avg=131072.0000 max=132384 max_rank=1 min=125399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=49305 max_load_reduction_pct=0.271370 moved_to_replicas=228800 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[180872, 229277, 98120, 88695, 41495, 117555, 168501, 124061] avg=131072.0000 max=229277 max_rank=1 min=41495 min_rank=4 imbalance_ratio=1.749245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] avg=131072.0000 max=132384 max_rank=1 min=125399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=49305 max_load_reduction_pct=0.271370 moved_to_replicas=228800 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[180872, 229277, 98120, 88695, 41495, 117555, 168501, 124061] avg=131072.0000 max=229277 max_rank=1 min=41495 min_rank=4 imbalance_ratio=1.749245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] avg=131072.0000 max=132385 max_rank=0 min=127413 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=96892 max_load_reduction_pct=0.422598 moved_to_replicas=366106 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] avg=131072.0000 max=132384 max_rank=1 min=125399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=49305 max_load_reduction_pct=0.271370 moved_to_replicas=228800 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[180872, 229277, 98120, 88695, 41495, 117555, 168501, 124061] avg=131072.0000 max=229277 max_rank=1 min=41495 min_rank=4 imbalance_ratio=1.749245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] avg=131072.0000 max=132385 max_rank=0 min=127413 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=96892 max_load_reduction_pct=0.422598 moved_to_replicas=366106 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[163221, 181689, 98306, 119751, 162662, 97931, 89560, 135456] avg=131072.0000 max=181689 max_rank=1 min=89560 min_rank=6 imbalance_ratio=1.386177 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] avg=131072.0000 max=132385 max_rank=0 min=127413 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=96892 max_load_reduction_pct=0.422598 moved_to_replicas=366106 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[180872, 229277, 98120, 88695, 41495, 117555, 168501, 124061] avg=131072.0000 max=229277 max_rank=1 min=41495 min_rank=4 imbalance_ratio=1.749245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[180872, 229277, 98120, 88695, 41495, 117555, 168501, 124061] avg=131072.0000 max=229277 max_rank=1 min=41495 min_rank=4 imbalance_ratio=1.749245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[163844, 32804, 98310, 140480, 141132, 118863, 130464, 222679] avg=131072.0000 max=222679 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.698906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] avg=131072.0000 max=132385 max_rank=0 min=127413 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=96892 max_load_reduction_pct=0.422598 moved_to_replicas=366106 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[132383, 132384, 129144, 132118, 132383, 132382, 125399, 132383] avg=131072.0000 max=132384 max_rank=1 min=125399 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=49305 max_load_reduction_pct=0.271370 moved_to_replicas=228800 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[180872, 229277, 98120, 88695, 41495, 117555, 168501, 124061] avg=131072.0000 max=229277 max_rank=1 min=41495 min_rank=4 imbalance_ratio=1.749245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] avg=131072.0000 max=132385 max_rank=0 min=127413 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=96892 max_load_reduction_pct=0.422598 moved_to_replicas=366106 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[163844, 32804, 98310, 140480, 141132, 118863, 130464, 222679] avg=131072.0000 max=222679 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.698906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] avg=131072.0000 max=132385 max_rank=0 min=127413 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=96892 max_load_reduction_pct=0.422598 moved_to_replicas=366106 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[163844, 32804, 98310, 140480, 141132, 118863, 130464, 222679] avg=131072.0000 max=222679 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.698906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] avg=131072.0000 max=132383 max_rank=0 min=127612 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90296 max_load_reduction_pct=0.405498 moved_to_replicas=274997 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] avg=131072.0000 max=132385 max_rank=0 min=127413 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=96892 max_load_reduction_pct=0.422598 moved_to_replicas=366106 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] avg=131072.0000 max=132383 max_rank=0 min=127612 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90296 max_load_reduction_pct=0.405498 moved_to_replicas=274997 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[163844, 32804, 98310, 140480, 141132, 118863, 130464, 222679] avg=131072.0000 max=222679 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.698906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[180872, 229277, 98120, 88695, 41495, 117555, 168501, 124061] avg=131072.0000 max=229277 max_rank=1 min=41495 min_rank=4 imbalance_ratio=1.749245 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] avg=131072.0000 max=132383 max_rank=0 min=127612 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90296 max_load_reduction_pct=0.405498 moved_to_replicas=274997 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[163844, 32804, 98310, 140480, 141132, 118863, 130464, 222679] avg=131072.0000 max=222679 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.698906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[163844, 32804, 98310, 140480, 141132, 118863, 130464, 222679] avg=131072.0000 max=222679 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.698906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[98035, 163488, 130814, 58047, 65497, 236575, 163808, 132312] avg=131072.0000 max=236575 max_rank=5 min=58047 min_rank=3 imbalance_ratio=1.804924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] avg=131072.0000 max=132383 max_rank=0 min=127612 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90296 max_load_reduction_pct=0.405498 moved_to_replicas=274997 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[163844, 32804, 98310, 140480, 141132, 118863, 130464, 222679] avg=131072.0000 max=222679 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.698906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[132385, 132383, 130884, 132382, 128361, 132385, 132383, 127413] avg=131072.0000 max=132385 max_rank=0 min=127413 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=96892 max_load_reduction_pct=0.422598 moved_to_replicas=366106 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] avg=131072.0000 max=132383 max_rank=0 min=127612 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90296 max_load_reduction_pct=0.405498 moved_to_replicas=274997 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[98035, 163488, 130814, 58047, 65497, 236575, 163808, 132312] avg=131072.0000 max=236575 max_rank=5 min=58047 min_rank=3 imbalance_ratio=1.804924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] avg=131072.0000 max=132383 max_rank=0 min=127612 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90296 max_load_reduction_pct=0.405498 moved_to_replicas=274997 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[98035, 163488, 130814, 58047, 65497, 236575, 163808, 132312] avg=131072.0000 max=236575 max_rank=5 min=58047 min_rank=3 imbalance_ratio=1.804924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] avg=131072.0000 max=132383 max_rank=0 min=126230 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104192 max_load_reduction_pct=0.440418 moved_to_replicas=328371 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] avg=131072.0000 max=132383 max_rank=0 min=127612 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90296 max_load_reduction_pct=0.405498 moved_to_replicas=274997 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] avg=131072.0000 max=132383 max_rank=0 min=126230 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104192 max_load_reduction_pct=0.440418 moved_to_replicas=328371 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[98035, 163488, 130814, 58047, 65497, 236575, 163808, 132312] avg=131072.0000 max=236575 max_rank=5 min=58047 min_rank=3 imbalance_ratio=1.804924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] avg=131072.0000 max=132383 max_rank=0 min=126230 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104192 max_load_reduction_pct=0.440418 moved_to_replicas=328371 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[163844, 32804, 98310, 140480, 141132, 118863, 130464, 222679] avg=131072.0000 max=222679 max_rank=7 min=32804 min_rank=1 imbalance_ratio=1.698906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[98035, 163488, 130814, 58047, 65497, 236575, 163808, 132312] avg=131072.0000 max=236575 max_rank=5 min=58047 min_rank=3 imbalance_ratio=1.804924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[98035, 163488, 130814, 58047, 65497, 236575, 163808, 132312] avg=131072.0000 max=236575 max_rank=5 min=58047 min_rank=3 imbalance_ratio=1.804924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[167958, 229463, 132497, 98314, 99373, 96040, 126612, 98319] avg=131072.0000 max=229463 max_rank=1 min=96040 min_rank=5 imbalance_ratio=1.750664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] avg=131072.0000 max=132383 max_rank=0 min=126230 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104192 max_load_reduction_pct=0.440418 moved_to_replicas=328371 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[98035, 163488, 130814, 58047, 65497, 236575, 163808, 132312] avg=131072.0000 max=236575 max_rank=5 min=58047 min_rank=3 imbalance_ratio=1.804924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[132383, 129798, 131172, 132382, 132383, 127612, 130464, 132382] avg=131072.0000 max=132383 max_rank=0 min=127612 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=90296 max_load_reduction_pct=0.405498 moved_to_replicas=274997 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[167958, 229463, 132497, 98314, 99373, 96040, 126612, 98319] avg=131072.0000 max=229463 max_rank=1 min=96040 min_rank=5 imbalance_ratio=1.750664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] avg=131072.0000 max=132383 max_rank=0 min=126230 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104192 max_load_reduction_pct=0.440418 moved_to_replicas=328371 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[167958, 229463, 132497, 98314, 99373, 96040, 126612, 98319] avg=131072.0000 max=229463 max_rank=1 min=96040 min_rank=5 imbalance_ratio=1.750664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] avg=131072.0000 max=132383 max_rank=0 min=126230 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104192 max_load_reduction_pct=0.440418 moved_to_replicas=328371 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] avg=131072.0000 max=132383 max_rank=0 min=128920 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97080 max_load_reduction_pct=0.423075 moved_to_replicas=265774 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] avg=131072.0000 max=132383 max_rank=0 min=126230 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104192 max_load_reduction_pct=0.440418 moved_to_replicas=328371 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] avg=131072.0000 max=132383 max_rank=0 min=128920 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97080 max_load_reduction_pct=0.423075 moved_to_replicas=265774 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] avg=131072.0000 max=132383 max_rank=0 min=128920 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97080 max_load_reduction_pct=0.423075 moved_to_replicas=265774 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[167958, 229463, 132497, 98314, 99373, 96040, 126612, 98319] avg=131072.0000 max=229463 max_rank=1 min=96040 min_rank=5 imbalance_ratio=1.750664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[98035, 163488, 130814, 58047, 65497, 236575, 163808, 132312] avg=131072.0000 max=236575 max_rank=5 min=58047 min_rank=3 imbalance_ratio=1.804924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[167958, 229463, 132497, 98314, 99373, 96040, 126612, 98319] avg=131072.0000 max=229463 max_rank=1 min=96040 min_rank=5 imbalance_ratio=1.750664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[167958, 229463, 132497, 98314, 99373, 96040, 126612, 98319] avg=131072.0000 max=229463 max_rank=1 min=96040 min_rank=5 imbalance_ratio=1.750664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[158495, 131304, 196140, 98799, 196609, 67132, 167311, 32786] avg=131072.0000 max=196609 max_rank=4 min=32786 min_rank=7 imbalance_ratio=1.500008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] avg=131072.0000 max=132383 max_rank=0 min=128920 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97080 max_load_reduction_pct=0.423075 moved_to_replicas=265774 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[158495, 131304, 196140, 98799, 196609, 67132, 167311, 32786] avg=131072.0000 max=196609 max_rank=4 min=32786 min_rank=7 imbalance_ratio=1.500008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[167958, 229463, 132497, 98314, 99373, 96040, 126612, 98319] avg=131072.0000 max=229463 max_rank=1 min=96040 min_rank=5 imbalance_ratio=1.750664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] avg=131072.0000 max=132383 max_rank=0 min=128920 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97080 max_load_reduction_pct=0.423075 moved_to_replicas=265774 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[158495, 131304, 196140, 98799, 196609, 67132, 167311, 32786] avg=131072.0000 max=196609 max_rank=4 min=32786 min_rank=7 imbalance_ratio=1.500008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[132383, 132383, 130814, 126230, 129688, 132383, 132383, 132312] avg=131072.0000 max=132383 max_rank=0 min=126230 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104192 max_load_reduction_pct=0.440418 moved_to_replicas=328371 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] avg=131072.0000 max=132385 max_rank=0 min=128003 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64224 max_load_reduction_pct=0.326658 moved_to_replicas=378636 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] avg=131072.0000 max=132383 max_rank=0 min=128920 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97080 max_load_reduction_pct=0.423075 moved_to_replicas=265774 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] avg=131072.0000 max=132385 max_rank=0 min=128003 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64224 max_load_reduction_pct=0.326658 moved_to_replicas=378636 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] avg=131072.0000 max=132383 max_rank=0 min=128920 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97080 max_load_reduction_pct=0.423075 moved_to_replicas=265774 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] avg=131072.0000 max=132385 max_rank=0 min=128003 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64224 max_load_reduction_pct=0.326658 moved_to_replicas=378636 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[158495, 131304, 196140, 98799, 196609, 67132, 167311, 32786] avg=131072.0000 max=196609 max_rank=4 min=32786 min_rank=7 imbalance_ratio=1.500008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[158495, 131304, 196140, 98799, 196609, 67132, 167311, 32786] avg=131072.0000 max=196609 max_rank=4 min=32786 min_rank=7 imbalance_ratio=1.500008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[167958, 229463, 132497, 98314, 99373, 96040, 126612, 98319] avg=131072.0000 max=229463 max_rank=1 min=96040 min_rank=5 imbalance_ratio=1.750664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[133623, 146636, 132322, 163862, 65577, 99881, 130970, 175705] avg=131072.0000 max=175705 max_rank=7 min=65577 min_rank=4 imbalance_ratio=1.340523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[158495, 131304, 196140, 98799, 196609, 67132, 167311, 32786] avg=131072.0000 max=196609 max_rank=4 min=32786 min_rank=7 imbalance_ratio=1.500008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] avg=131072.0000 max=132385 max_rank=0 min=128003 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64224 max_load_reduction_pct=0.326658 moved_to_replicas=378636 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[133623, 146636, 132322, 163862, 65577, 99881, 130970, 175705] avg=131072.0000 max=175705 max_rank=7 min=65577 min_rank=4 imbalance_ratio=1.340523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[158495, 131304, 196140, 98799, 196609, 67132, 167311, 32786] avg=131072.0000 max=196609 max_rank=4 min=32786 min_rank=7 imbalance_ratio=1.500008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] avg=131072.0000 max=132385 max_rank=0 min=128003 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64224 max_load_reduction_pct=0.326658 moved_to_replicas=378636 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[133623, 146636, 132322, 163862, 65577, 99881, 130970, 175705] avg=131072.0000 max=175705 max_rank=7 min=65577 min_rank=4 imbalance_ratio=1.340523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[132383, 132383, 132383, 131079, 132139, 128920, 129421, 129868] avg=131072.0000 max=132383 max_rank=0 min=128920 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=97080 max_load_reduction_pct=0.423075 moved_to_replicas=265774 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] avg=131072.0000 max=132383 max_rank=0 min=124393 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=43322 max_load_reduction_pct=0.246561 moved_to_replicas=178720 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] avg=131072.0000 max=132385 max_rank=0 min=128003 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64224 max_load_reduction_pct=0.326658 moved_to_replicas=378636 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] avg=131072.0000 max=132383 max_rank=0 min=124393 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=43322 max_load_reduction_pct=0.246561 moved_to_replicas=178720 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] avg=131072.0000 max=132385 max_rank=0 min=128003 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64224 max_load_reduction_pct=0.326658 moved_to_replicas=378636 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[133623, 146636, 132322, 163862, 65577, 99881, 130970, 175705] avg=131072.0000 max=175705 max_rank=7 min=65577 min_rank=4 imbalance_ratio=1.340523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] avg=131072.0000 max=132383 max_rank=0 min=124393 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=43322 max_load_reduction_pct=0.246561 moved_to_replicas=178720 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[133623, 146636, 132322, 163862, 65577, 99881, 130970, 175705] avg=131072.0000 max=175705 max_rank=7 min=65577 min_rank=4 imbalance_ratio=1.340523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[158495, 131304, 196140, 98799, 196609, 67132, 167311, 32786] avg=131072.0000 max=196609 max_rank=4 min=32786 min_rank=7 imbalance_ratio=1.500008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[98216, 231006, 196423, 65557, 131607, 65552, 96582, 163633] avg=131072.0000 max=231006 max_rank=1 min=65552 min_rank=5 imbalance_ratio=1.762436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] avg=131072.0000 max=132383 max_rank=0 min=124393 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=43322 max_load_reduction_pct=0.246561 moved_to_replicas=178720 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[98216, 231006, 196423, 65557, 131607, 65552, 96582, 163633] avg=131072.0000 max=231006 max_rank=1 min=65552 min_rank=5 imbalance_ratio=1.762436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[133623, 146636, 132322, 163862, 65577, 99881, 130970, 175705] avg=131072.0000 max=175705 max_rank=7 min=65577 min_rank=4 imbalance_ratio=1.340523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[133623, 146636, 132322, 163862, 65577, 99881, 130970, 175705] avg=131072.0000 max=175705 max_rank=7 min=65577 min_rank=4 imbalance_ratio=1.340523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[98216, 231006, 196423, 65557, 131607, 65552, 96582, 163633] avg=131072.0000 max=231006 max_rank=1 min=65552 min_rank=5 imbalance_ratio=1.762436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] avg=131072.0000 max=132383 max_rank=0 min=124393 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=43322 max_load_reduction_pct=0.246561 moved_to_replicas=178720 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[132385, 131304, 132383, 131565, 132383, 128170, 132383, 128003] avg=131072.0000 max=132385 max_rank=0 min=128003 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64224 max_load_reduction_pct=0.326658 moved_to_replicas=378636 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98623 max_load_reduction_pct=0.426928 moved_to_replicas=387045 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98623 max_load_reduction_pct=0.426928 moved_to_replicas=387045 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] avg=131072.0000 max=132383 max_rank=0 min=124393 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=43322 max_load_reduction_pct=0.246561 moved_to_replicas=178720 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] avg=131072.0000 max=132383 max_rank=0 min=124393 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=43322 max_load_reduction_pct=0.246561 moved_to_replicas=178720 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98623 max_load_reduction_pct=0.426928 moved_to_replicas=387045 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[98216, 231006, 196423, 65557, 131607, 65552, 96582, 163633] avg=131072.0000 max=231006 max_rank=1 min=65552 min_rank=5 imbalance_ratio=1.762436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[98216, 231006, 196423, 65557, 131607, 65552, 96582, 163633] avg=131072.0000 max=231006 max_rank=1 min=65552 min_rank=5 imbalance_ratio=1.762436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[133623, 146636, 132322, 163862, 65577, 99881, 130970, 175705] avg=131072.0000 max=175705 max_rank=7 min=65577 min_rank=4 imbalance_ratio=1.340523 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65665, 66006, 163857, 131059, 164105, 130658, 130578] avg=131072.0000 max=196648 max_rank=0 min=65665 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65665, 66006, 163857, 131059, 164105, 130658, 130578] avg=131072.0000 max=196648 max_rank=0 min=65665 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98623 max_load_reduction_pct=0.426928 moved_to_replicas=387045 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[98216, 231006, 196423, 65557, 131607, 65552, 96582, 163633] avg=131072.0000 max=231006 max_rank=1 min=65552 min_rank=5 imbalance_ratio=1.762436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[98216, 231006, 196423, 65557, 131607, 65552, 96582, 163633] avg=131072.0000 max=231006 max_rank=1 min=65552 min_rank=5 imbalance_ratio=1.762436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65665, 66006, 163857, 131059, 164105, 130658, 130578] avg=131072.0000 max=196648 max_rank=0 min=65665 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98623 max_load_reduction_pct=0.426928 moved_to_replicas=387045 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] avg=131072.0000 max=132383 max_rank=0 min=129227 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254641 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[132383, 132383, 132322, 132383, 124393, 131360, 130970, 132382] avg=131072.0000 max=132383 max_rank=0 min=124393 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=43322 max_load_reduction_pct=0.246561 moved_to_replicas=178720 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] avg=131072.0000 max=132383 max_rank=0 min=129227 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254641 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98623 max_load_reduction_pct=0.426928 moved_to_replicas=387045 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98623 max_load_reduction_pct=0.426928 moved_to_replicas=387045 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] avg=131072.0000 max=132383 max_rank=0 min=129227 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254641 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65665, 66006, 163857, 131059, 164105, 130658, 130578] avg=131072.0000 max=196648 max_rank=0 min=65665 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65665, 66006, 163857, 131059, 164105, 130658, 130578] avg=131072.0000 max=196648 max_rank=0 min=65665 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[131545, 98352, 130846, 163021, 163763, 130783, 67722, 162544] avg=131072.0000 max=163763 max_rank=4 min=67722 min_rank=6 imbalance_ratio=1.249413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[98216, 231006, 196423, 65557, 131607, 65552, 96582, 163633] avg=131072.0000 max=231006 max_rank=1 min=65552 min_rank=5 imbalance_ratio=1.762436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[131545, 98352, 130846, 163021, 163763, 130783, 67722, 162544] avg=131072.0000 max=163763 max_rank=4 min=67722 min_rank=6 imbalance_ratio=1.249413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] avg=131072.0000 max=132383 max_rank=0 min=129227 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254641 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65665, 66006, 163857, 131059, 164105, 130658, 130578] avg=131072.0000 max=196648 max_rank=0 min=65665 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65665, 66006, 163857, 131059, 164105, 130658, 130578] avg=131072.0000 max=196648 max_rank=0 min=65665 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[131545, 98352, 130846, 163021, 163763, 130783, 67722, 162544] avg=131072.0000 max=163763 max_rank=4 min=67722 min_rank=6 imbalance_ratio=1.249413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] avg=131072.0000 max=132383 max_rank=0 min=129227 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254641 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] avg=131072.0000 max=132383 max_rank=3 min=128990 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31380 max_load_reduction_pct=0.191618 moved_to_replicas=182662 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[129792, 132383, 132383, 129597, 131607, 131084, 129347, 132383] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=98623 max_load_reduction_pct=0.426928 moved_to_replicas=387045 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] avg=131072.0000 max=132383 max_rank=3 min=128990 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31380 max_load_reduction_pct=0.191618 moved_to_replicas=182662 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] avg=131072.0000 max=132383 max_rank=0 min=129227 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254641 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] avg=131072.0000 max=132383 max_rank=0 min=129227 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254641 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] avg=131072.0000 max=132383 max_rank=3 min=128990 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31380 max_load_reduction_pct=0.191618 moved_to_replicas=182662 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[131545, 98352, 130846, 163021, 163763, 130783, 67722, 162544] avg=131072.0000 max=163763 max_rank=4 min=67722 min_rank=6 imbalance_ratio=1.249413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[131545, 98352, 130846, 163021, 163763, 130783, 67722, 162544] avg=131072.0000 max=163763 max_rank=4 min=67722 min_rank=6 imbalance_ratio=1.249413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[163854, 98343, 98306, 163805, 163817, 65466, 163476, 131509] avg=131072.0000 max=163854 max_rank=0 min=65466 min_rank=5 imbalance_ratio=1.250107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[196648, 65665, 66006, 163857, 131059, 164105, 130658, 130578] avg=131072.0000 max=196648 max_rank=0 min=65665 min_rank=1 imbalance_ratio=1.500305 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[163854, 98343, 98306, 163805, 163817, 65466, 163476, 131509] avg=131072.0000 max=163854 max_rank=0 min=65466 min_rank=5 imbalance_ratio=1.250107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] avg=131072.0000 max=132383 max_rank=3 min=128990 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31380 max_load_reduction_pct=0.191618 moved_to_replicas=182662 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[131545, 98352, 130846, 163021, 163763, 130783, 67722, 162544] avg=131072.0000 max=163763 max_rank=4 min=67722 min_rank=6 imbalance_ratio=1.249413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[163854, 98343, 98306, 163805, 163817, 65466, 163476, 131509] avg=131072.0000 max=163854 max_rank=0 min=65466 min_rank=5 imbalance_ratio=1.250107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[131545, 98352, 130846, 163021, 163763, 130783, 67722, 162544] avg=131072.0000 max=163763 max_rank=4 min=67722 min_rank=6 imbalance_ratio=1.249413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] avg=131072.0000 max=132383 max_rank=3 min=128990 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31380 max_load_reduction_pct=0.191618 moved_to_replicas=182662 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] avg=131072.0000 max=132383 max_rank=0 min=128371 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31471 max_load_reduction_pct=0.192067 moved_to_replicas=250053 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[132383, 129905, 129227, 132383, 131059, 132383, 130658, 130578] avg=131072.0000 max=132383 max_rank=0 min=129227 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64265 max_load_reduction_pct=0.326802 moved_to_replicas=254641 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] avg=131072.0000 max=132383 max_rank=0 min=128371 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31471 max_load_reduction_pct=0.192067 moved_to_replicas=250053 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] avg=131072.0000 max=132383 max_rank=3 min=128990 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31380 max_load_reduction_pct=0.191618 moved_to_replicas=182662 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] avg=131072.0000 max=132383 max_rank=0 min=128371 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31471 max_load_reduction_pct=0.192067 moved_to_replicas=250053 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] avg=131072.0000 max=132383 max_rank=3 min=128990 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31380 max_load_reduction_pct=0.191618 moved_to_replicas=182662 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[163854, 98343, 98306, 163805, 163817, 65466, 163476, 131509] avg=131072.0000 max=163854 max_rank=0 min=65466 min_rank=5 imbalance_ratio=1.250107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[228150, 98087, 132301, 196022, 65703, 130910, 873, 196530] avg=131072.0000 max=228150 max_rank=0 min=873 min_rank=6 imbalance_ratio=1.740646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[163854, 98343, 98306, 163805, 163817, 65466, 163476, 131509] avg=131072.0000 max=163854 max_rank=0 min=65466 min_rank=5 imbalance_ratio=1.250107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[131545, 98352, 130846, 163021, 163763, 130783, 67722, 162544] avg=131072.0000 max=163763 max_rank=4 min=67722 min_rank=6 imbalance_ratio=1.249413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[228150, 98087, 132301, 196022, 65703, 130910, 873, 196530] avg=131072.0000 max=228150 max_rank=0 min=873 min_rank=6 imbalance_ratio=1.740646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[163854, 98343, 98306, 163805, 163817, 65466, 163476, 131509] avg=131072.0000 max=163854 max_rank=0 min=65466 min_rank=5 imbalance_ratio=1.250107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] avg=131072.0000 max=132383 max_rank=0 min=128371 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31471 max_load_reduction_pct=0.192067 moved_to_replicas=250053 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[228150, 98087, 132301, 196022, 65703, 130910, 873, 196530] avg=131072.0000 max=228150 max_rank=0 min=873 min_rank=6 imbalance_ratio=1.740646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[163854, 98343, 98306, 163805, 163817, 65466, 163476, 131509] avg=131072.0000 max=163854 max_rank=0 min=65466 min_rank=5 imbalance_ratio=1.250107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] avg=131072.0000 max=132383 max_rank=0 min=127319 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95767 max_load_reduction_pct=0.419755 moved_to_replicas=444313 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] avg=131072.0000 max=132383 max_rank=0 min=128371 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31471 max_load_reduction_pct=0.192067 moved_to_replicas=250053 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] avg=131072.0000 max=132383 max_rank=0 min=127319 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95767 max_load_reduction_pct=0.419755 moved_to_replicas=444313 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[131545, 128990, 130846, 132383, 132383, 130783, 129263, 132383] avg=131072.0000 max=132383 max_rank=3 min=128990 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=31380 max_load_reduction_pct=0.191618 moved_to_replicas=182662 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] avg=131072.0000 max=132383 max_rank=0 min=128371 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31471 max_load_reduction_pct=0.192067 moved_to_replicas=250053 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] avg=131072.0000 max=132383 max_rank=0 min=127319 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95767 max_load_reduction_pct=0.419755 moved_to_replicas=444313 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] avg=131072.0000 max=132383 max_rank=0 min=128371 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31471 max_load_reduction_pct=0.192067 moved_to_replicas=250053 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[228150, 98087, 132301, 196022, 65703, 130910, 873, 196530] avg=131072.0000 max=228150 max_rank=0 min=873 min_rank=6 imbalance_ratio=1.740646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[228150, 98087, 132301, 196022, 65703, 130910, 873, 196530] avg=131072.0000 max=228150 max_rank=0 min=873 min_rank=6 imbalance_ratio=1.740646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[163854, 98343, 98306, 163805, 163817, 65466, 163476, 131509] avg=131072.0000 max=163854 max_rank=0 min=65466 min_rank=5 imbalance_ratio=1.250107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[228150, 98087, 132301, 196022, 65703, 130910, 873, 196530] avg=131072.0000 max=228150 max_rank=0 min=873 min_rank=6 imbalance_ratio=1.740646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] avg=131072.0000 max=132383 max_rank=0 min=127319 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95767 max_load_reduction_pct=0.419755 moved_to_replicas=444313 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[228150, 98087, 132301, 196022, 65703, 130910, 873, 196530] avg=131072.0000 max=228150 max_rank=0 min=873 min_rank=6 imbalance_ratio=1.740646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] avg=131072.0000 max=132383 max_rank=0 min=127319 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95767 max_load_reduction_pct=0.419755 moved_to_replicas=444313 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[132383, 129436, 129728, 132383, 132383, 128371, 132383, 131509] avg=131072.0000 max=132383 max_rank=0 min=128371 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=31471 max_load_reduction_pct=0.192067 moved_to_replicas=250053 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] avg=131072.0000 max=132383 max_rank=0 min=127319 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95767 max_load_reduction_pct=0.419755 moved_to_replicas=444313 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] avg=131072.0000 max=132383 max_rank=0 min=127319 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95767 max_load_reduction_pct=0.419755 moved_to_replicas=444313 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[228150, 98087, 132301, 196022, 65703, 130910, 873, 196530] avg=131072.0000 max=228150 max_rank=0 min=873 min_rank=6 imbalance_ratio=1.740646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[132383, 130853, 132301, 132383, 127319, 130910, 130044, 132383] avg=131072.0000 max=132383 max_rank=0 min=127319 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=95767 max_load_reduction_pct=0.419755 moved_to_replicas=444313 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=4176565 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1030272 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=4179447 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1026304 peak_demand=132386 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=4205677 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1000832 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=4221840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=984832 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=4189293 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016192 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=4190218 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014912 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=4195158 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1010560 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=4196234 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1008896 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:17:53] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 572.8 | throughput per GPU (TFLOP/s/GPU): 582.0 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.033488E+01 | loss scale: 1.0 | grad norm: 17.374 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=163840 expected_recv=255672 expected_overflow=0 exact_recv_per_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=163840 expected_recv=260345 expected_overflow=0 exact_recv_per_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=163840 expected_recv=255672 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=163840 expected_recv=260345 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=257792 slack_slots=69888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=163840 expected_recv=257300 expected_overflow=0 exact_recv_per_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=163840 expected_recv=257300 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=163840 expected_recv=260914 expected_overflow=0 exact_recv_per_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=163840 expected_recv=256168 expected_overflow=0 exact_recv_per_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=259840 slack_slots=67840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=163840 expected_recv=260914 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=163840 expected_recv=257956 expected_overflow=0 exact_recv_per_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=163840 expected_recv=256168 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=163840 expected_recv=257956 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=259328 slack_slots=68352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=260480 slack_slots=67200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=163840 expected_recv=263054 expected_overflow=0 exact_recv_per_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=163840 expected_recv=255013 expected_overflow=0 exact_recv_per_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=163840 expected_recv=263054 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=163840 expected_recv=255013 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262418 expected_overflow=0 exact_recv_per_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=163840 expected_recv=258554 expected_overflow=0 exact_recv_per_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=257920 slack_slots=69760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262418 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=163840 expected_recv=258554 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=163840 expected_recv=258077 expected_overflow=0 exact_recv_per_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=163840 expected_recv=258077 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=163840 expected_recv=261576 expected_overflow=0 exact_recv_per_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262099 expected_overflow=0 exact_recv_per_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=163840 expected_recv=260157 expected_overflow=0 exact_recv_per_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262134 expected_overflow=0 exact_recv_per_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=163840 expected_recv=261576 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262099 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=163840 expected_recv=260157 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262134 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=163840 expected_recv=262312 expected_overflow=0 exact_recv_per_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=163840 expected_recv=262312 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=163840 expected_recv=258901 expected_overflow=0 exact_recv_per_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=163840 expected_recv=257912 expected_overflow=0 exact_recv_per_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=163840 expected_recv=258901 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=163840 expected_recv=260910 expected_overflow=0 exact_recv_per_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=163840 expected_recv=257912 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=163840 expected_recv=258960 expected_overflow=0 exact_recv_per_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=260352 slack_slots=67328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=163840 expected_recv=260910 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=163840 expected_recv=258960 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=163840 expected_recv=256720 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=163840 expected_recv=256720 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=259328 slack_slots=68352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=163840 expected_recv=259544 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=163840 expected_recv=259544 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=163840 expected_recv=257059 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=163840 expected_recv=256687 expected_overflow=0 exact_recv_per_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=163840 expected_recv=256687 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=163840 expected_recv=257059 actual_valid_recv=260224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=163840 expected_recv=251869 expected_overflow=0 exact_recv_per_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=260480 slack_slots=67200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=260224 slack_slots=67456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=163840 expected_recv=251869 actual_valid_recv=255232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=255232 slack_slots=72448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=163840 expected_recv=260268 expected_overflow=0 exact_recv_per_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262027 expected_overflow=0 exact_recv_per_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=163840 expected_recv=260268 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262027 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=163840 expected_recv=259886 expected_overflow=0 exact_recv_per_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=163840 expected_recv=261329 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=163840 expected_recv=259886 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=163840 expected_recv=261329 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=163840 expected_recv=260732 expected_overflow=0 exact_recv_per_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=163840 expected_recv=256834 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=163840 expected_recv=260732 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=163840 expected_recv=256834 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=163840 expected_recv=258907 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=259200 slack_slots=68480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=163840 expected_recv=258907 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=260736 slack_slots=66944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=163840 expected_recv=259866 expected_overflow=0 exact_recv_per_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=163840 expected_recv=259939 expected_overflow=0 exact_recv_per_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=163840 expected_recv=259866 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=163840 expected_recv=259939 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=163840 expected_recv=258829 expected_overflow=0 exact_recv_per_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=163840 expected_recv=259976 expected_overflow=0 exact_recv_per_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=163840 expected_recv=258829 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=260736 slack_slots=66944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=163840 expected_recv=259976 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=163840 expected_recv=257426 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=163840 expected_recv=259417 expected_overflow=0 exact_recv_per_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=163840 expected_recv=261015 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=163840 expected_recv=257426 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=163840 expected_recv=259417 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=259840 slack_slots=67840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=163840 expected_recv=261015 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=163840 expected_recv=259432 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=163840 expected_recv=259432 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=163840 expected_recv=260751 expected_overflow=0 exact_recv_per_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=163840 expected_recv=260751 actual_valid_recv=262656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=262656 slack_slots=65024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=163840 expected_recv=260532 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=163840 expected_recv=261988 expected_overflow=0 exact_recv_per_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=163840 expected_recv=261921 expected_overflow=0 exact_recv_per_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=163840 expected_recv=260532 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=163840 expected_recv=261988 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262199 expected_overflow=0 exact_recv_per_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=163840 expected_recv=261921 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262199 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=163840 expected_recv=256905 expected_overflow=0 exact_recv_per_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=163840 expected_recv=256905 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262245 expected_overflow=0 exact_recv_per_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=163840 expected_recv=260697 expected_overflow=0 exact_recv_per_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=259840 slack_slots=67840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262245 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=163840 expected_recv=260697 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=163840 expected_recv=261915 expected_overflow=0 exact_recv_per_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=163840 expected_recv=259680 expected_overflow=0 exact_recv_per_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=163840 expected_recv=261915 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=163840 expected_recv=259680 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=163840 expected_recv=257222 expected_overflow=0 exact_recv_per_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262046 expected_overflow=0 exact_recv_per_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=163840 expected_recv=257222 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=163840 expected_recv=263602 expected_overflow=0 exact_recv_per_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=259456 slack_slots=68224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262046 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=163840 expected_recv=263602 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265600 slack_slots=62080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=163840 expected_recv=257766 expected_overflow=0 exact_recv_per_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=163840 expected_recv=257766 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=163840 expected_recv=256892 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=163840 expected_recv=259346 expected_overflow=0 exact_recv_per_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=163840 expected_recv=256892 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262919 expected_overflow=0 exact_recv_per_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=163840 expected_recv=259346 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=259072 slack_slots=68608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262919 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262216 expected_overflow=0 exact_recv_per_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=163840 expected_recv=259170 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=163840 expected_recv=259234 expected_overflow=0 exact_recv_per_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262216 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=163840 expected_recv=259170 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=163840 expected_recv=259234 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=163840 expected_recv=260608 expected_overflow=0 exact_recv_per_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=163840 expected_recv=260608 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262129 expected_overflow=0 exact_recv_per_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[83428, 119795, 143912, 140954, 117610, 188331, 150518, 104028] avg=131072.0000 max=188331 max_rank=5 min=83428 min_rank=0 imbalance_ratio=1.436852 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262129 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262501 expected_overflow=0 exact_recv_per_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=163840 expected_recv=257260 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262501 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=163840 expected_recv=257260 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] avg=131072.0000 max=132385 max_rank=5 min=128365 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=55946 max_load_reduction_pct=0.297062 moved_to_replicas=190267 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[83428, 119795, 143912, 140954, 117610, 188331, 150518, 104028] avg=131072.0000 max=188331 max_rank=5 min=83428 min_rank=0 imbalance_ratio=1.436852 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=259712 slack_slots=67968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=163840 expected_recv=259612 expected_overflow=0 exact_recv_per_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=163840 expected_recv=259612 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] avg=131072.0000 max=132385 max_rank=5 min=128365 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=55946 max_load_reduction_pct=0.297062 moved_to_replicas=190267 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[83428, 119795, 143912, 140954, 117610, 188331, 150518, 104028] avg=131072.0000 max=188331 max_rank=5 min=83428 min_rank=0 imbalance_ratio=1.436852 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[83428, 119795, 143912, 140954, 117610, 188331, 150518, 104028] avg=131072.0000 max=188331 max_rank=5 min=83428 min_rank=0 imbalance_ratio=1.436852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[194041, 98405, 195992, 130457, 16352, 147287, 168723, 97319] avg=131072.0000 max=195992 max_rank=2 min=16352 min_rank=4 imbalance_ratio=1.495300 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=163840 expected_recv=259378 expected_overflow=0 exact_recv_per_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[83428, 119795, 143912, 140954, 117610, 188331, 150518, 104028] avg=131072.0000 max=188331 max_rank=5 min=83428 min_rank=0 imbalance_ratio=1.436852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[83428, 119795, 143912, 140954, 117610, 188331, 150518, 104028] avg=131072.0000 max=188331 max_rank=5 min=83428 min_rank=0 imbalance_ratio=1.436852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] avg=131072.0000 max=132385 max_rank=5 min=128365 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=55946 max_load_reduction_pct=0.297062 moved_to_replicas=190267 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=163840 expected_recv=259378 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] avg=131072.0000 max=132385 max_rank=5 min=128365 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=55946 max_load_reduction_pct=0.297062 moved_to_replicas=190267 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[194041, 98405, 195992, 130457, 16352, 147287, 168723, 97319] avg=131072.0000 max=195992 max_rank=2 min=16352 min_rank=4 imbalance_ratio=1.495300 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] avg=131072.0000 max=132383 max_rank=0 min=126214 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=63609 max_load_reduction_pct=0.324549 moved_to_replicas=348873 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[83428, 119795, 143912, 140954, 117610, 188331, 150518, 104028] avg=131072.0000 max=188331 max_rank=5 min=83428 min_rank=0 imbalance_ratio=1.436852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] avg=131072.0000 max=132385 max_rank=5 min=128365 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=55946 max_load_reduction_pct=0.297062 moved_to_replicas=190267 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] avg=131072.0000 max=132385 max_rank=5 min=128365 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=55946 max_load_reduction_pct=0.297062 moved_to_replicas=190267 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] avg=131072.0000 max=132383 max_rank=0 min=126214 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=63609 max_load_reduction_pct=0.324549 moved_to_replicas=348873 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[194041, 98405, 195992, 130457, 16352, 147287, 168723, 97319] avg=131072.0000 max=195992 max_rank=2 min=16352 min_rank=4 imbalance_ratio=1.495300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[194041, 98405, 195992, 130457, 16352, 147287, 168723, 97319] avg=131072.0000 max=195992 max_rank=2 min=16352 min_rank=4 imbalance_ratio=1.495300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] avg=131072.0000 max=132385 max_rank=5 min=128365 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=55946 max_load_reduction_pct=0.297062 moved_to_replicas=190267 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[104040, 131153, 153428, 146671, 98277, 170448, 195907, 48652] avg=131072.0000 max=195907 max_rank=6 min=48652 min_rank=7 imbalance_ratio=1.494652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[83428, 119795, 143912, 140954, 117610, 188331, 150518, 104028] avg=131072.0000 max=188331 max_rank=5 min=83428 min_rank=0 imbalance_ratio=1.436852 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[194041, 98405, 195992, 130457, 16352, 147287, 168723, 97319] avg=131072.0000 max=195992 max_rank=2 min=16352 min_rank=4 imbalance_ratio=1.495300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[194041, 98405, 195992, 130457, 16352, 147287, 168723, 97319] avg=131072.0000 max=195992 max_rank=2 min=16352 min_rank=4 imbalance_ratio=1.495300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] avg=131072.0000 max=132383 max_rank=0 min=126214 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=63609 max_load_reduction_pct=0.324549 moved_to_replicas=348873 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] avg=131072.0000 max=132383 max_rank=0 min=126214 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=63609 max_load_reduction_pct=0.324549 moved_to_replicas=348873 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] avg=131072.0000 max=132385 max_rank=2 min=126463 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63522 max_load_reduction_pct=0.324246 moved_to_replicas=271880 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[104040, 131153, 153428, 146671, 98277, 170448, 195907, 48652] avg=131072.0000 max=195907 max_rank=6 min=48652 min_rank=7 imbalance_ratio=1.494652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[132384, 128365, 132383, 132384, 129381, 132385, 132382, 128912] avg=131072.0000 max=132385 max_rank=5 min=128365 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=55946 max_load_reduction_pct=0.297062 moved_to_replicas=190267 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[194041, 98405, 195992, 130457, 16352, 147287, 168723, 97319] avg=131072.0000 max=195992 max_rank=2 min=16352 min_rank=4 imbalance_ratio=1.495300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] avg=131072.0000 max=132383 max_rank=0 min=126214 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=63609 max_load_reduction_pct=0.324549 moved_to_replicas=348873 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] avg=131072.0000 max=132383 max_rank=0 min=126214 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=63609 max_load_reduction_pct=0.324549 moved_to_replicas=348873 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[104040, 131153, 153428, 146671, 98277, 170448, 195907, 48652] avg=131072.0000 max=195907 max_rank=6 min=48652 min_rank=7 imbalance_ratio=1.494652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] avg=131072.0000 max=132385 max_rank=2 min=126463 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63522 max_load_reduction_pct=0.324246 moved_to_replicas=271880 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[104040, 131153, 153428, 146671, 98277, 170448, 195907, 48652] avg=131072.0000 max=195907 max_rank=6 min=48652 min_rank=7 imbalance_ratio=1.494652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[98223, 98491, 65181, 196647, 98990, 196368, 163614, 131062] avg=131072.0000 max=196647 max_rank=3 min=65181 min_rank=2 imbalance_ratio=1.500298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] avg=131072.0000 max=132383 max_rank=0 min=126214 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=63609 max_load_reduction_pct=0.324549 moved_to_replicas=348873 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] avg=131072.0000 max=132385 max_rank=2 min=126463 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63522 max_load_reduction_pct=0.324246 moved_to_replicas=271880 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[104040, 131153, 153428, 146671, 98277, 170448, 195907, 48652] avg=131072.0000 max=195907 max_rank=6 min=48652 min_rank=7 imbalance_ratio=1.494652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[194041, 98405, 195992, 130457, 16352, 147287, 168723, 97319] avg=131072.0000 max=195992 max_rank=2 min=16352 min_rank=4 imbalance_ratio=1.495300 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[104040, 131153, 153428, 146671, 98277, 170448, 195907, 48652] avg=131072.0000 max=195907 max_rank=6 min=48652 min_rank=7 imbalance_ratio=1.494652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] avg=131072.0000 max=132385 max_rank=2 min=126463 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63522 max_load_reduction_pct=0.324246 moved_to_replicas=271880 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] avg=131072.0000 max=132383 max_rank=3 min=129445 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64264 max_load_reduction_pct=0.326799 moved_to_replicas=319033 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[98223, 98491, 65181, 196647, 98990, 196368, 163614, 131062] avg=131072.0000 max=196647 max_rank=3 min=65181 min_rank=2 imbalance_ratio=1.500298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[104040, 131153, 153428, 146671, 98277, 170448, 195907, 48652] avg=131072.0000 max=195907 max_rank=6 min=48652 min_rank=7 imbalance_ratio=1.494652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] avg=131072.0000 max=132385 max_rank=2 min=126463 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63522 max_load_reduction_pct=0.324246 moved_to_replicas=271880 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[132383, 131142, 132383, 130457, 131233, 132381, 132383, 126214] avg=131072.0000 max=132383 max_rank=0 min=126214 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=63609 max_load_reduction_pct=0.324549 moved_to_replicas=348873 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] avg=131072.0000 max=132385 max_rank=2 min=126463 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63522 max_load_reduction_pct=0.324246 moved_to_replicas=271880 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[98223, 98491, 65181, 196647, 98990, 196368, 163614, 131062] avg=131072.0000 max=196647 max_rank=3 min=65181 min_rank=2 imbalance_ratio=1.500298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] avg=131072.0000 max=132383 max_rank=3 min=129445 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64264 max_load_reduction_pct=0.326799 moved_to_replicas=319033 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[98223, 98491, 65181, 196647, 98990, 196368, 163614, 131062] avg=131072.0000 max=196647 max_rank=3 min=65181 min_rank=2 imbalance_ratio=1.500298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65614, 227271, 131146, 158675, 178465, 122955, 66080, 98370] avg=131072.0000 max=227271 max_rank=1 min=65614 min_rank=0 imbalance_ratio=1.733940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] avg=131072.0000 max=132385 max_rank=2 min=126463 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63522 max_load_reduction_pct=0.324246 moved_to_replicas=271880 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] avg=131072.0000 max=132383 max_rank=3 min=129445 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64264 max_load_reduction_pct=0.326799 moved_to_replicas=319033 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[98223, 98491, 65181, 196647, 98990, 196368, 163614, 131062] avg=131072.0000 max=196647 max_rank=3 min=65181 min_rank=2 imbalance_ratio=1.500298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] avg=131072.0000 max=132384 max_rank=3 min=127735 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=94887 max_load_reduction_pct=0.417506 moved_to_replicas=336964 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] avg=131072.0000 max=132383 max_rank=3 min=129445 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64264 max_load_reduction_pct=0.326799 moved_to_replicas=319033 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[98223, 98491, 65181, 196647, 98990, 196368, 163614, 131062] avg=131072.0000 max=196647 max_rank=3 min=65181 min_rank=2 imbalance_ratio=1.500298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[104040, 131153, 153428, 146671, 98277, 170448, 195907, 48652] avg=131072.0000 max=195907 max_rank=6 min=48652 min_rank=7 imbalance_ratio=1.494652 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65614, 227271, 131146, 158675, 178465, 122955, 66080, 98370] avg=131072.0000 max=227271 max_rank=1 min=65614 min_rank=0 imbalance_ratio=1.733940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] avg=131072.0000 max=132383 max_rank=3 min=129445 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64264 max_load_reduction_pct=0.326799 moved_to_replicas=319033 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[98223, 98491, 65181, 196647, 98990, 196368, 163614, 131062] avg=131072.0000 max=196647 max_rank=3 min=65181 min_rank=2 imbalance_ratio=1.500298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65614, 227271, 131146, 158675, 178465, 122955, 66080, 98370] avg=131072.0000 max=227271 max_rank=1 min=65614 min_rank=0 imbalance_ratio=1.733940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] avg=131072.0000 max=132383 max_rank=3 min=129445 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64264 max_load_reduction_pct=0.326799 moved_to_replicas=319033 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[130385, 131153, 132385, 132384, 131040, 132383, 132383, 126463] avg=131072.0000 max=132385 max_rank=2 min=126463 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=63522 max_load_reduction_pct=0.324246 moved_to_replicas=271880 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] avg=131072.0000 max=132384 max_rank=3 min=127735 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=94887 max_load_reduction_pct=0.417506 moved_to_replicas=336964 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[163827, 196565, 155976, 66315, 98286, 135288, 166751, 65568] avg=131072.0000 max=196565 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65614, 227271, 131146, 158675, 178465, 122955, 66080, 98370] avg=131072.0000 max=227271 max_rank=1 min=65614 min_rank=0 imbalance_ratio=1.733940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] avg=131072.0000 max=132383 max_rank=3 min=129445 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64264 max_load_reduction_pct=0.326799 moved_to_replicas=319033 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] avg=131072.0000 max=132384 max_rank=3 min=127735 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=94887 max_load_reduction_pct=0.417506 moved_to_replicas=336964 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65614, 227271, 131146, 158675, 178465, 122955, 66080, 98370] avg=131072.0000 max=227271 max_rank=1 min=65614 min_rank=0 imbalance_ratio=1.733940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] avg=131072.0000 max=132384 max_rank=2 min=126391 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64181 max_load_reduction_pct=0.326513 moved_to_replicas=313133 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] avg=131072.0000 max=132384 max_rank=3 min=127735 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=94887 max_load_reduction_pct=0.417506 moved_to_replicas=336964 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65614, 227271, 131146, 158675, 178465, 122955, 66080, 98370] avg=131072.0000 max=227271 max_rank=1 min=65614 min_rank=0 imbalance_ratio=1.733940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[163827, 196565, 155976, 66315, 98286, 135288, 166751, 65568] avg=131072.0000 max=196565 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[98223, 98491, 65181, 196647, 98990, 196368, 163614, 131062] avg=131072.0000 max=196647 max_rank=3 min=65181 min_rank=2 imbalance_ratio=1.500298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] avg=131072.0000 max=132384 max_rank=3 min=127735 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=94887 max_load_reduction_pct=0.417506 moved_to_replicas=336964 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65614, 227271, 131146, 158675, 178465, 122955, 66080, 98370] avg=131072.0000 max=227271 max_rank=1 min=65614 min_rank=0 imbalance_ratio=1.733940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[163827, 196565, 155976, 66315, 98286, 135288, 166751, 65568] avg=131072.0000 max=196565 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] avg=131072.0000 max=132384 max_rank=3 min=127735 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=94887 max_load_reduction_pct=0.417506 moved_to_replicas=336964 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] avg=131072.0000 max=132384 max_rank=2 min=126391 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64181 max_load_reduction_pct=0.326513 moved_to_replicas=313133 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[163751, 32802, 78843, 170425, 196400, 139612, 39898, 226845] avg=131072.0000 max=226845 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.730690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[130986, 129713, 129445, 132383, 130221, 132383, 132383, 131062] avg=131072.0000 max=132383 max_rank=3 min=129445 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64264 max_load_reduction_pct=0.326799 moved_to_replicas=319033 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[163827, 196565, 155976, 66315, 98286, 135288, 166751, 65568] avg=131072.0000 max=196565 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] avg=131072.0000 max=132384 max_rank=3 min=127735 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=94887 max_load_reduction_pct=0.417506 moved_to_replicas=336964 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] avg=131072.0000 max=132384 max_rank=2 min=126391 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64181 max_load_reduction_pct=0.326513 moved_to_replicas=313133 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] avg=131072.0000 max=132385 max_rank=5 min=127012 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=94460 max_load_reduction_pct=0.416408 moved_to_replicas=475933 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[163827, 196565, 155976, 66315, 98286, 135288, 166751, 65568] avg=131072.0000 max=196565 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] avg=131072.0000 max=132384 max_rank=2 min=126391 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64181 max_load_reduction_pct=0.326513 moved_to_replicas=313133 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[163827, 196565, 155976, 66315, 98286, 135288, 166751, 65568] avg=131072.0000 max=196565 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[163751, 32802, 78843, 170425, 196400, 139612, 39898, 226845] avg=131072.0000 max=226845 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.730690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65614, 227271, 131146, 158675, 178465, 122955, 66080, 98370] avg=131072.0000 max=227271 max_rank=1 min=65614 min_rank=0 imbalance_ratio=1.733940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[163827, 196565, 155976, 66315, 98286, 135288, 166751, 65568] avg=131072.0000 max=196565 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[163751, 32802, 78843, 170425, 196400, 139612, 39898, 226845] avg=131072.0000 max=226845 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.730690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] avg=131072.0000 max=132384 max_rank=2 min=126391 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64181 max_load_reduction_pct=0.326513 moved_to_replicas=313133 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[64133, 163930, 153846, 98299, 76463, 295120, 98447, 98338] avg=131072.0000 max=295120 max_rank=5 min=64133 min_rank=0 imbalance_ratio=2.251587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] avg=131072.0000 max=132384 max_rank=2 min=126391 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64181 max_load_reduction_pct=0.326513 moved_to_replicas=313133 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] avg=131072.0000 max=132385 max_rank=5 min=127012 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=94460 max_load_reduction_pct=0.416408 moved_to_replicas=475933 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[131137, 132383, 131146, 132384, 132382, 129025, 132384, 127735] avg=131072.0000 max=132384 max_rank=3 min=127735 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=94887 max_load_reduction_pct=0.417506 moved_to_replicas=336964 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[163751, 32802, 78843, 170425, 196400, 139612, 39898, 226845] avg=131072.0000 max=226845 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.730690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] avg=131072.0000 max=132385 max_rank=5 min=127012 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=94460 max_load_reduction_pct=0.416408 moved_to_replicas=475933 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] avg=131072.0000 max=132384 max_rank=2 min=126391 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64181 max_load_reduction_pct=0.326513 moved_to_replicas=313133 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] avg=131072.0000 max=132385 max_rank=2 min=129652 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=162735 max_load_reduction_pct=0.551420 moved_to_replicas=429043 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] avg=131072.0000 max=132385 max_rank=5 min=127012 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=94460 max_load_reduction_pct=0.416408 moved_to_replicas=475933 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[163751, 32802, 78843, 170425, 196400, 139612, 39898, 226845] avg=131072.0000 max=226845 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.730690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[163751, 32802, 78843, 170425, 196400, 139612, 39898, 226845] avg=131072.0000 max=226845 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.730690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[64133, 163930, 153846, 98299, 76463, 295120, 98447, 98338] avg=131072.0000 max=295120 max_rank=5 min=64133 min_rank=0 imbalance_ratio=2.251587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[163827, 196565, 155976, 66315, 98286, 135288, 166751, 65568] avg=131072.0000 max=196565 max_rank=1 min=65568 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[64133, 163930, 153846, 98299, 76463, 295120, 98447, 98338] avg=131072.0000 max=295120 max_rank=5 min=64133 min_rank=0 imbalance_ratio=2.251587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] avg=131072.0000 max=132385 max_rank=5 min=127012 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=94460 max_load_reduction_pct=0.416408 moved_to_replicas=475933 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[159735, 228056, 130879, 65746, 98407, 65562, 159697, 140494] avg=131072.0000 max=228056 max_rank=1 min=65562 min_rank=5 imbalance_ratio=1.739929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] avg=131072.0000 max=132385 max_rank=5 min=127012 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=94460 max_load_reduction_pct=0.416408 moved_to_replicas=475933 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[163751, 32802, 78843, 170425, 196400, 139612, 39898, 226845] avg=131072.0000 max=226845 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.730690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] avg=131072.0000 max=132385 max_rank=2 min=129652 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=162735 max_load_reduction_pct=0.551420 moved_to_replicas=429043 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[132383, 132383, 132384, 130497, 126391, 132383, 132383, 129772] avg=131072.0000 max=132384 max_rank=2 min=126391 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64181 max_load_reduction_pct=0.326513 moved_to_replicas=313133 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[64133, 163930, 153846, 98299, 76463, 295120, 98447, 98338] avg=131072.0000 max=295120 max_rank=5 min=64133 min_rank=0 imbalance_ratio=2.251587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] avg=131072.0000 max=132385 max_rank=2 min=129652 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=162735 max_load_reduction_pct=0.551420 moved_to_replicas=429043 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=127307 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=95671 max_load_reduction_pct=0.419507 moved_to_replicas=317224 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] avg=131072.0000 max=132385 max_rank=5 min=127012 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=94460 max_load_reduction_pct=0.416408 moved_to_replicas=475933 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[64133, 163930, 153846, 98299, 76463, 295120, 98447, 98338] avg=131072.0000 max=295120 max_rank=5 min=64133 min_rank=0 imbalance_ratio=2.251587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] avg=131072.0000 max=132385 max_rank=2 min=129652 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=162735 max_load_reduction_pct=0.551420 moved_to_replicas=429043 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[64133, 163930, 153846, 98299, 76463, 295120, 98447, 98338] avg=131072.0000 max=295120 max_rank=5 min=64133 min_rank=0 imbalance_ratio=2.251587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[159735, 228056, 130879, 65746, 98407, 65562, 159697, 140494] avg=131072.0000 max=228056 max_rank=1 min=65562 min_rank=5 imbalance_ratio=1.739929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[163751, 32802, 78843, 170425, 196400, 139612, 39898, 226845] avg=131072.0000 max=226845 max_rank=7 min=32802 min_rank=1 imbalance_ratio=1.730690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[159735, 228056, 130879, 65746, 98407, 65562, 159697, 140494] avg=131072.0000 max=228056 max_rank=1 min=65562 min_rank=5 imbalance_ratio=1.739929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] avg=131072.0000 max=132385 max_rank=2 min=129652 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=162735 max_load_reduction_pct=0.551420 moved_to_replicas=429043 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[98027, 163938, 98326, 98305, 262125, 163702, 164043, 110] avg=131072.0000 max=262125 max_rank=4 min=110 min_rank=7 imbalance_ratio=1.999855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] avg=131072.0000 max=132385 max_rank=2 min=129652 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=162735 max_load_reduction_pct=0.551420 moved_to_replicas=429043 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=127307 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=95671 max_load_reduction_pct=0.419507 moved_to_replicas=317224 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[64133, 163930, 153846, 98299, 76463, 295120, 98447, 98338] avg=131072.0000 max=295120 max_rank=5 min=64133 min_rank=0 imbalance_ratio=2.251587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[159735, 228056, 130879, 65746, 98407, 65562, 159697, 140494] avg=131072.0000 max=228056 max_rank=1 min=65562 min_rank=5 imbalance_ratio=1.739929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[132383, 127264, 132383, 132383, 132383, 132385, 127012, 132383] avg=131072.0000 max=132385 max_rank=5 min=127012 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=94460 max_load_reduction_pct=0.416408 moved_to_replicas=475933 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=127307 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=95671 max_load_reduction_pct=0.419507 moved_to_replicas=317224 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] avg=131072.0000 max=132383 max_rank=1 min=129493 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129742 max_load_reduction_pct=0.494962 moved_to_replicas=448633 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] avg=131072.0000 max=132385 max_rank=2 min=129652 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=162735 max_load_reduction_pct=0.551420 moved_to_replicas=429043 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=127307 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=95671 max_load_reduction_pct=0.419507 moved_to_replicas=317224 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[159735, 228056, 130879, 65746, 98407, 65562, 159697, 140494] avg=131072.0000 max=228056 max_rank=1 min=65562 min_rank=5 imbalance_ratio=1.739929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[159735, 228056, 130879, 65746, 98407, 65562, 159697, 140494] avg=131072.0000 max=228056 max_rank=1 min=65562 min_rank=5 imbalance_ratio=1.739929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[98027, 163938, 98326, 98305, 262125, 163702, 164043, 110] avg=131072.0000 max=262125 max_rank=4 min=110 min_rank=7 imbalance_ratio=1.999855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[64133, 163930, 153846, 98299, 76463, 295120, 98447, 98338] avg=131072.0000 max=295120 max_rank=5 min=64133 min_rank=0 imbalance_ratio=2.251587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[98027, 163938, 98326, 98305, 262125, 163702, 164043, 110] avg=131072.0000 max=262125 max_rank=4 min=110 min_rank=7 imbalance_ratio=1.999855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[98302, 229312, 83036, 113418, 97987, 163845, 98804, 163872] avg=131072.0000 max=229312 max_rank=1 min=83036 min_rank=2 imbalance_ratio=1.749512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=127307 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=95671 max_load_reduction_pct=0.419507 moved_to_replicas=317224 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=127307 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=95671 max_load_reduction_pct=0.419507 moved_to_replicas=317224 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] avg=131072.0000 max=132383 max_rank=1 min=129493 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129742 max_load_reduction_pct=0.494962 moved_to_replicas=448633 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[159735, 228056, 130879, 65746, 98407, 65562, 159697, 140494] avg=131072.0000 max=228056 max_rank=1 min=65562 min_rank=5 imbalance_ratio=1.739929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[98027, 163938, 98326, 98305, 262125, 163702, 164043, 110] avg=131072.0000 max=262125 max_rank=4 min=110 min_rank=7 imbalance_ratio=1.999855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[129652, 132383, 132385, 131057, 130683, 132383, 129994, 130039] avg=131072.0000 max=132385 max_rank=2 min=129652 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=162735 max_load_reduction_pct=0.551420 moved_to_replicas=429043 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] avg=131072.0000 max=132383 max_rank=1 min=129493 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129742 max_load_reduction_pct=0.494962 moved_to_replicas=448633 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] avg=131072.0000 max=132383 max_rank=1 min=128292 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96929 max_load_reduction_pct=0.422695 moved_to_replicas=319877 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=127307 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=95671 max_load_reduction_pct=0.419507 moved_to_replicas=317224 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] avg=131072.0000 max=132383 max_rank=1 min=129493 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129742 max_load_reduction_pct=0.494962 moved_to_replicas=448633 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[98027, 163938, 98326, 98305, 262125, 163702, 164043, 110] avg=131072.0000 max=262125 max_rank=4 min=110 min_rank=7 imbalance_ratio=1.999855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[98027, 163938, 98326, 98305, 262125, 163702, 164043, 110] avg=131072.0000 max=262125 max_rank=4 min=110 min_rank=7 imbalance_ratio=1.999855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[98302, 229312, 83036, 113418, 97987, 163845, 98804, 163872] avg=131072.0000 max=229312 max_rank=1 min=83036 min_rank=2 imbalance_ratio=1.749512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[98302, 229312, 83036, 113418, 97987, 163845, 98804, 163872] avg=131072.0000 max=229312 max_rank=1 min=83036 min_rank=2 imbalance_ratio=1.749512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[159735, 228056, 130879, 65746, 98407, 65562, 159697, 140494] avg=131072.0000 max=228056 max_rank=1 min=65562 min_rank=5 imbalance_ratio=1.739929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[130973, 233663, 130732, 65582, 131724, 130973, 126581, 98348] avg=131072.0000 max=233663 max_rank=1 min=65582 min_rank=3 imbalance_ratio=1.782707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] avg=131072.0000 max=132383 max_rank=1 min=129493 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129742 max_load_reduction_pct=0.494962 moved_to_replicas=448633 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] avg=131072.0000 max=132383 max_rank=1 min=129493 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129742 max_load_reduction_pct=0.494962 moved_to_replicas=448633 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] avg=131072.0000 max=132383 max_rank=1 min=128292 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96929 max_load_reduction_pct=0.422695 moved_to_replicas=319877 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[98027, 163938, 98326, 98305, 262125, 163702, 164043, 110] avg=131072.0000 max=262125 max_rank=4 min=110 min_rank=7 imbalance_ratio=1.999855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[98302, 229312, 83036, 113418, 97987, 163845, 98804, 163872] avg=131072.0000 max=229312 max_rank=1 min=83036 min_rank=2 imbalance_ratio=1.749512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] avg=131072.0000 max=132383 max_rank=1 min=128292 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96929 max_load_reduction_pct=0.422695 moved_to_replicas=319877 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] avg=131072.0000 max=132383 max_rank=1 min=129591 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101280 max_load_reduction_pct=0.433445 moved_to_replicas=203407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[132384, 132383, 130879, 127307, 132380, 128475, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=127307 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=95671 max_load_reduction_pct=0.419507 moved_to_replicas=317224 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] avg=131072.0000 max=132383 max_rank=1 min=128292 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96929 max_load_reduction_pct=0.422695 moved_to_replicas=319877 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] avg=131072.0000 max=132383 max_rank=1 min=129493 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129742 max_load_reduction_pct=0.494962 moved_to_replicas=448633 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[98302, 229312, 83036, 113418, 97987, 163845, 98804, 163872] avg=131072.0000 max=229312 max_rank=1 min=83036 min_rank=2 imbalance_ratio=1.749512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[98302, 229312, 83036, 113418, 97987, 163845, 98804, 163872] avg=131072.0000 max=229312 max_rank=1 min=83036 min_rank=2 imbalance_ratio=1.749512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[130973, 233663, 130732, 65582, 131724, 130973, 126581, 98348] avg=131072.0000 max=233663 max_rank=1 min=65582 min_rank=3 imbalance_ratio=1.782707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[130973, 233663, 130732, 65582, 131724, 130973, 126581, 98348] avg=131072.0000 max=233663 max_rank=1 min=65582 min_rank=3 imbalance_ratio=1.782707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[163658, 67813, 65589, 161765, 164007, 130951, 163748, 131045] avg=131072.0000 max=164007 max_rank=4 min=65589 min_rank=2 imbalance_ratio=1.251274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[98027, 163938, 98326, 98305, 262125, 163702, 164043, 110] avg=131072.0000 max=262125 max_rank=4 min=110 min_rank=7 imbalance_ratio=1.999855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] avg=131072.0000 max=132383 max_rank=1 min=128292 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96929 max_load_reduction_pct=0.422695 moved_to_replicas=319877 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] avg=131072.0000 max=132383 max_rank=1 min=128292 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96929 max_load_reduction_pct=0.422695 moved_to_replicas=319877 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] avg=131072.0000 max=132383 max_rank=1 min=129591 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101280 max_load_reduction_pct=0.433445 moved_to_replicas=203407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[130973, 233663, 130732, 65582, 131724, 130973, 126581, 98348] avg=131072.0000 max=233663 max_rank=1 min=65582 min_rank=3 imbalance_ratio=1.782707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[98302, 229312, 83036, 113418, 97987, 163845, 98804, 163872] avg=131072.0000 max=229312 max_rank=1 min=83036 min_rank=2 imbalance_ratio=1.749512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] avg=131072.0000 max=132383 max_rank=1 min=129591 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101280 max_load_reduction_pct=0.433445 moved_to_replicas=203407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] avg=131072.0000 max=132383 max_rank=0 min=128488 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31624 max_load_reduction_pct=0.192821 moved_to_replicas=248768 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[129493, 132383, 129881, 129965, 132383, 132383, 132383, 129705] avg=131072.0000 max=132383 max_rank=1 min=129493 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=129742 max_load_reduction_pct=0.494962 moved_to_replicas=448633 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] avg=131072.0000 max=132383 max_rank=1 min=129591 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101280 max_load_reduction_pct=0.433445 moved_to_replicas=203407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] avg=131072.0000 max=132383 max_rank=1 min=128292 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96929 max_load_reduction_pct=0.422695 moved_to_replicas=319877 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[130973, 233663, 130732, 65582, 131724, 130973, 126581, 98348] avg=131072.0000 max=233663 max_rank=1 min=65582 min_rank=3 imbalance_ratio=1.782707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[130973, 233663, 130732, 65582, 131724, 130973, 126581, 98348] avg=131072.0000 max=233663 max_rank=1 min=65582 min_rank=3 imbalance_ratio=1.782707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[163658, 67813, 65589, 161765, 164007, 130951, 163748, 131045] avg=131072.0000 max=164007 max_rank=4 min=65589 min_rank=2 imbalance_ratio=1.251274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[163658, 67813, 65589, 161765, 164007, 130951, 163748, 131045] avg=131072.0000 max=164007 max_rank=4 min=65589 min_rank=2 imbalance_ratio=1.251274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[127086, 98303, 229267, 134182, 97456, 196588, 65577, 100117] avg=131072.0000 max=229267 max_rank=2 min=65577 min_rank=6 imbalance_ratio=1.749168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] avg=131072.0000 max=132383 max_rank=1 min=129591 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101280 max_load_reduction_pct=0.433445 moved_to_replicas=203407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[98302, 229312, 83036, 113418, 97987, 163845, 98804, 163872] avg=131072.0000 max=229312 max_rank=1 min=83036 min_rank=2 imbalance_ratio=1.749512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] avg=131072.0000 max=132383 max_rank=1 min=129591 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101280 max_load_reduction_pct=0.433445 moved_to_replicas=203407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] avg=131072.0000 max=132383 max_rank=0 min=128488 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31624 max_load_reduction_pct=0.192821 moved_to_replicas=248768 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[163658, 67813, 65589, 161765, 164007, 130951, 163748, 131045] avg=131072.0000 max=164007 max_rank=4 min=65589 min_rank=2 imbalance_ratio=1.251274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[130973, 233663, 130732, 65582, 131724, 130973, 126581, 98348] avg=131072.0000 max=233663 max_rank=1 min=65582 min_rank=3 imbalance_ratio=1.782707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] avg=131072.0000 max=132383 max_rank=0 min=128488 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31624 max_load_reduction_pct=0.192821 moved_to_replicas=248768 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] avg=131072.0000 max=132383 max_rank=2 min=128885 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96884 max_load_reduction_pct=0.422582 moved_to_replicas=328246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[129714, 132383, 128292, 132383, 130745, 132383, 130293, 132383] avg=131072.0000 max=132383 max_rank=1 min=128292 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96929 max_load_reduction_pct=0.422695 moved_to_replicas=319877 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] avg=131072.0000 max=132383 max_rank=0 min=128488 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31624 max_load_reduction_pct=0.192821 moved_to_replicas=248768 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] avg=131072.0000 max=132383 max_rank=1 min=129591 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101280 max_load_reduction_pct=0.433445 moved_to_replicas=203407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[163658, 67813, 65589, 161765, 164007, 130951, 163748, 131045] avg=131072.0000 max=164007 max_rank=4 min=65589 min_rank=2 imbalance_ratio=1.251274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[163658, 67813, 65589, 161765, 164007, 130951, 163748, 131045] avg=131072.0000 max=164007 max_rank=4 min=65589 min_rank=2 imbalance_ratio=1.251274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[127086, 98303, 229267, 134182, 97456, 196588, 65577, 100117] avg=131072.0000 max=229267 max_rank=2 min=65577 min_rank=6 imbalance_ratio=1.749168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[127086, 98303, 229267, 134182, 97456, 196588, 65577, 100117] avg=131072.0000 max=229267 max_rank=2 min=65577 min_rank=6 imbalance_ratio=1.749168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[229497, 133510, 32811, 65607, 32907, 161394, 229249, 163601] avg=131072.0000 max=229497 max_rank=0 min=32811 min_rank=2 imbalance_ratio=1.750923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] avg=131072.0000 max=132383 max_rank=0 min=128488 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31624 max_load_reduction_pct=0.192821 moved_to_replicas=248768 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[130973, 233663, 130732, 65582, 131724, 130973, 126581, 98348] avg=131072.0000 max=233663 max_rank=1 min=65582 min_rank=3 imbalance_ratio=1.782707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] avg=131072.0000 max=132383 max_rank=0 min=128488 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31624 max_load_reduction_pct=0.192821 moved_to_replicas=248768 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[127086, 98303, 229267, 134182, 97456, 196588, 65577, 100117] avg=131072.0000 max=229267 max_rank=2 min=65577 min_rank=6 imbalance_ratio=1.749168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] avg=131072.0000 max=132383 max_rank=2 min=128885 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96884 max_load_reduction_pct=0.422582 moved_to_replicas=328246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[163658, 67813, 65589, 161765, 164007, 130951, 163748, 131045] avg=131072.0000 max=164007 max_rank=4 min=65589 min_rank=2 imbalance_ratio=1.251274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] avg=131072.0000 max=132383 max_rank=2 min=128885 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96884 max_load_reduction_pct=0.422582 moved_to_replicas=328246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128528 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=97114 max_load_reduction_pct=0.423160 moved_to_replicas=510496 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[130973, 132383, 130732, 131096, 131724, 130973, 129591, 131104] avg=131072.0000 max=132383 max_rank=1 min=129591 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=101280 max_load_reduction_pct=0.433445 moved_to_replicas=203407 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] avg=131072.0000 max=132383 max_rank=2 min=128885 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96884 max_load_reduction_pct=0.422582 moved_to_replicas=328246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] avg=131072.0000 max=132383 max_rank=0 min=128488 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31624 max_load_reduction_pct=0.192821 moved_to_replicas=248768 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[127086, 98303, 229267, 134182, 97456, 196588, 65577, 100117] avg=131072.0000 max=229267 max_rank=2 min=65577 min_rank=6 imbalance_ratio=1.749168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[127086, 98303, 229267, 134182, 97456, 196588, 65577, 100117] avg=131072.0000 max=229267 max_rank=2 min=65577 min_rank=6 imbalance_ratio=1.749168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[229497, 133510, 32811, 65607, 32907, 161394, 229249, 163601] avg=131072.0000 max=229497 max_rank=0 min=32811 min_rank=2 imbalance_ratio=1.750923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[229497, 133510, 32811, 65607, 32907, 161394, 229249, 163601] avg=131072.0000 max=229497 max_rank=0 min=32811 min_rank=2 imbalance_ratio=1.750923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131249, 98280, 131067, 65538, 196451, 65531, 196668] avg=131072.0000 max=196668 max_rank=7 min=65531 min_rank=6 imbalance_ratio=1.500458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] avg=131072.0000 max=132383 max_rank=2 min=128885 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96884 max_load_reduction_pct=0.422582 moved_to_replicas=328246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[163658, 67813, 65589, 161765, 164007, 130951, 163748, 131045] avg=131072.0000 max=164007 max_rank=4 min=65589 min_rank=2 imbalance_ratio=1.251274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] avg=131072.0000 max=132383 max_rank=2 min=128885 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96884 max_load_reduction_pct=0.422582 moved_to_replicas=328246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128528 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=97114 max_load_reduction_pct=0.423160 moved_to_replicas=510496 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[229497, 133510, 32811, 65607, 32907, 161394, 229249, 163601] avg=131072.0000 max=229497 max_rank=0 min=32811 min_rank=2 imbalance_ratio=1.750923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[127086, 98303, 229267, 134182, 97456, 196588, 65577, 100117] avg=131072.0000 max=229267 max_rank=2 min=65577 min_rank=6 imbalance_ratio=1.749168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128528 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=97114 max_load_reduction_pct=0.423160 moved_to_replicas=510496 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] avg=131072.0000 max=132383 max_rank=0 min=129592 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64285 max_load_reduction_pct=0.326871 moved_to_replicas=319503 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[132383, 128560, 128488, 132383, 132383, 130951, 132383, 131045] avg=131072.0000 max=132383 max_rank=0 min=128488 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=31624 max_load_reduction_pct=0.192821 moved_to_replicas=248768 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[229497, 133510, 32811, 65607, 32907, 161394, 229249, 163601] avg=131072.0000 max=229497 max_rank=0 min=32811 min_rank=2 imbalance_ratio=1.750923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128528 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=97114 max_load_reduction_pct=0.423160 moved_to_replicas=510496 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] avg=131072.0000 max=132383 max_rank=2 min=128885 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96884 max_load_reduction_pct=0.422582 moved_to_replicas=328246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[229497, 133510, 32811, 65607, 32907, 161394, 229249, 163601] avg=131072.0000 max=229497 max_rank=0 min=32811 min_rank=2 imbalance_ratio=1.750923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131249, 98280, 131067, 65538, 196451, 65531, 196668] avg=131072.0000 max=196668 max_rank=7 min=65531 min_rank=6 imbalance_ratio=1.500458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131249, 98280, 131067, 65538, 196451, 65531, 196668] avg=131072.0000 max=196668 max_rank=7 min=65531 min_rank=6 imbalance_ratio=1.500458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128528 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=97114 max_load_reduction_pct=0.423160 moved_to_replicas=510496 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[127086, 98303, 229267, 134182, 97456, 196588, 65577, 100117] avg=131072.0000 max=229267 max_rank=2 min=65577 min_rank=6 imbalance_ratio=1.749168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128528 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=97114 max_load_reduction_pct=0.423160 moved_to_replicas=510496 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] avg=131072.0000 max=132383 max_rank=0 min=129592 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64285 max_load_reduction_pct=0.326871 moved_to_replicas=319503 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] avg=131072.0000 max=132383 max_rank=0 min=129592 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64285 max_load_reduction_pct=0.326871 moved_to_replicas=319503 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131249, 98280, 131067, 65538, 196451, 65531, 196668] avg=131072.0000 max=196668 max_rank=7 min=65531 min_rank=6 imbalance_ratio=1.500458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[229497, 133510, 32811, 65607, 32907, 161394, 229249, 163601] avg=131072.0000 max=229497 max_rank=0 min=32811 min_rank=2 imbalance_ratio=1.750923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[128885, 129677, 132383, 132383, 130210, 132383, 131089, 131566] avg=131072.0000 max=132383 max_rank=2 min=128885 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=96884 max_load_reduction_pct=0.422582 moved_to_replicas=328246 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131249, 98280, 131067, 65538, 196451, 65531, 196668] avg=131072.0000 max=196668 max_rank=7 min=65531 min_rank=6 imbalance_ratio=1.500458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] avg=131072.0000 max=132383 max_rank=0 min=129592 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64285 max_load_reduction_pct=0.326871 moved_to_replicas=319503 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131249, 98280, 131067, 65538, 196451, 65531, 196668] avg=131072.0000 max=196668 max_rank=7 min=65531 min_rank=6 imbalance_ratio=1.500458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128528 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=97114 max_load_reduction_pct=0.423160 moved_to_replicas=510496 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] avg=131072.0000 max=132383 max_rank=0 min=129592 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64285 max_load_reduction_pct=0.326871 moved_to_replicas=319503 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[229497, 133510, 32811, 65607, 32907, 161394, 229249, 163601] avg=131072.0000 max=229497 max_rank=0 min=32811 min_rank=2 imbalance_ratio=1.750923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] avg=131072.0000 max=132383 max_rank=0 min=129592 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64285 max_load_reduction_pct=0.326871 moved_to_replicas=319503 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131249, 98280, 131067, 65538, 196451, 65531, 196668] avg=131072.0000 max=196668 max_rank=7 min=65531 min_rank=6 imbalance_ratio=1.500458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[132383, 132383, 128528, 129579, 128554, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128528 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=97114 max_load_reduction_pct=0.423160 moved_to_replicas=510496 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] avg=131072.0000 max=132383 max_rank=0 min=129592 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64285 max_load_reduction_pct=0.326871 moved_to_replicas=319503 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[163792, 131249, 98280, 131067, 65538, 196451, 65531, 196668] avg=131072.0000 max=196668 max_rank=7 min=65531 min_rank=6 imbalance_ratio=1.500458 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[132383, 131249, 129592, 131067, 129823, 132383, 129696, 132383] avg=131072.0000 max=132383 max_rank=0 min=129592 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64285 max_load_reduction_pct=0.326871 moved_to_replicas=319503 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=4186158 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018368 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=4216830 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=989440 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=4200175 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1005312 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=4190844 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016064 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=4187839 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017600 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=4188300 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016448 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=4202300 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1002368 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=4181986 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1021824 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:17:54] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 583.8 | throughput per GPU (TFLOP/s/GPU): 571.1 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.016673E+01 | loss scale: 1.0 | grad norm: 17.755 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=163840 expected_recv=254740 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=163840 expected_recv=258631 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=163840 expected_recv=254740 actual_valid_recv=257152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=163840 expected_recv=259947 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=257152 slack_slots=70528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=163840 expected_recv=258631 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=163840 expected_recv=259947 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=163840 expected_recv=255974 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=163840 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=163840 expected_recv=255974 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=163840 expected_recv=263689 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=163840 expected_recv=264763 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=259072 slack_slots=68608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=163840 expected_recv=263689 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=163840 expected_recv=253665 expected_overflow=0 exact_recv_per_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=163840 expected_recv=253665 actual_valid_recv=257280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=257280 slack_slots=70400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=163840 expected_recv=264094 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=163840 expected_recv=258314 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=163840 expected_recv=264094 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=163840 expected_recv=258314 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=163840 expected_recv=253259 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262421 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=163840 expected_recv=253259 actual_valid_recv=256256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=163840 expected_recv=260524 expected_overflow=0 exact_recv_per_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=256256 slack_slots=71424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262421 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=163840 expected_recv=260524 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=163840 expected_recv=254150 expected_overflow=0 exact_recv_per_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=163840 expected_recv=258650 expected_overflow=0 exact_recv_per_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=163840 expected_recv=254150 actual_valid_recv=256640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=163840 expected_recv=259078 expected_overflow=0 exact_recv_per_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=268160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=163840 expected_recv=258650 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=256640 slack_slots=71040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=268160 slack_slots=59520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=163840 expected_recv=259078 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=163840 expected_recv=258558 expected_overflow=0 exact_recv_per_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=163840 expected_recv=258558 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262275 expected_overflow=0 exact_recv_per_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=163840 expected_recv=258177 expected_overflow=0 exact_recv_per_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262275 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=163840 expected_recv=258177 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=163840 expected_recv=261776 expected_overflow=0 exact_recv_per_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=163840 expected_recv=261776 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=163840 expected_recv=259947 expected_overflow=0 exact_recv_per_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=163840 expected_recv=260627 expected_overflow=0 exact_recv_per_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=163840 expected_recv=259947 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=163840 expected_recv=260627 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=163840 expected_recv=261453 expected_overflow=0 exact_recv_per_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=163840 expected_recv=259051 expected_overflow=0 exact_recv_per_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=163840 expected_recv=261453 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=163840 expected_recv=261901 expected_overflow=0 exact_recv_per_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=163840 expected_recv=259051 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=163840 expected_recv=261901 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=261760 slack_slots=65920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=163840 expected_recv=261195 expected_overflow=0 exact_recv_per_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262700 expected_overflow=0 exact_recv_per_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=163840 expected_recv=261195 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262700 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=163840 expected_recv=254782 expected_overflow=0 exact_recv_per_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262273 expected_overflow=0 exact_recv_per_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=163840 expected_recv=262179 expected_overflow=0 exact_recv_per_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=163840 expected_recv=254782 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262273 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=163840 expected_recv=262179 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=257792 slack_slots=69888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=163840 expected_recv=254527 expected_overflow=0 exact_recv_per_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=163840 expected_recv=254527 actual_valid_recv=257152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=257152 slack_slots=70528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=163840 expected_recv=256612 expected_overflow=0 exact_recv_per_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=163840 expected_recv=256612 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=163840 expected_recv=260993 expected_overflow=0 exact_recv_per_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262073 expected_overflow=0 exact_recv_per_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=259712 slack_slots=67968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=163840 expected_recv=260993 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=163840 expected_recv=261103 expected_overflow=0 exact_recv_per_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262073 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=163840 expected_recv=261103 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=163840 expected_recv=259583 expected_overflow=0 exact_recv_per_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=163840 expected_recv=259583 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=163840 expected_recv=259102 expected_overflow=0 exact_recv_per_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262105 expected_overflow=0 exact_recv_per_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=163840 expected_recv=261827 expected_overflow=0 exact_recv_per_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=163840 expected_recv=259102 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262105 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=163840 expected_recv=257074 expected_overflow=0 exact_recv_per_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=163840 expected_recv=261827 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=261760 slack_slots=65920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=163840 expected_recv=257074 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=259840 slack_slots=67840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262115 expected_overflow=0 exact_recv_per_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262115 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=163840 expected_recv=259732 expected_overflow=0 exact_recv_per_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=163840 expected_recv=262252 expected_overflow=0 exact_recv_per_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=163840 expected_recv=259732 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=163840 expected_recv=262252 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=163840 expected_recv=254245 expected_overflow=0 exact_recv_per_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267904 slack_slots=59776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=163840 expected_recv=254245 actual_valid_recv=256256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=163840 expected_recv=256825 expected_overflow=0 exact_recv_per_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=256256 slack_slots=71424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=163840 expected_recv=256825 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=259200 slack_slots=68480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=163840 expected_recv=258880 expected_overflow=0 exact_recv_per_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=163840 expected_recv=258481 expected_overflow=0 exact_recv_per_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=163840 expected_recv=258880 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=163840 expected_recv=258481 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=163840 expected_recv=260073 expected_overflow=0 exact_recv_per_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=260864 slack_slots=66816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=163840 expected_recv=260654 expected_overflow=0 exact_recv_per_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=163840 expected_recv=260073 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=163840 expected_recv=260654 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=163840 expected_recv=263428 expected_overflow=0 exact_recv_per_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=163840 expected_recv=259766 expected_overflow=0 exact_recv_per_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=163840 expected_recv=257904 expected_overflow=0 exact_recv_per_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=163840 expected_recv=263428 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=163840 expected_recv=259766 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=163840 expected_recv=257904 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262102 expected_overflow=0 exact_recv_per_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=163840 expected_recv=259654 expected_overflow=0 exact_recv_per_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=260608 slack_slots=67072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262102 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=163840 expected_recv=259654 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=163840 expected_recv=254664 expected_overflow=0 exact_recv_per_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=163840 expected_recv=254664 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=257792 slack_slots=69888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=163840 expected_recv=259185 expected_overflow=0 exact_recv_per_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=163840 expected_recv=259185 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262112 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=163840 expected_recv=259473 expected_overflow=0 exact_recv_per_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262110 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262112 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=163840 expected_recv=259473 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262110 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=163840 expected_recv=259469 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262011 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=163840 expected_recv=259469 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262011 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262363 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262363 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=163840 expected_recv=259555 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=163840 expected_recv=256877 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=163840 expected_recv=259555 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=163840 expected_recv=256877 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=259200 slack_slots=68480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=163840 expected_recv=259464 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=163840 expected_recv=259464 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[95947, 149052, 148466, 166795, 107665, 147373, 105920, 127358] avg=131072.0000 max=166795 max_rank=3 min=95947 min_rank=0 imbalance_ratio=1.272545 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=163840 expected_recv=256981 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[95947, 149052, 148466, 166795, 107665, 147373, 105920, 127358] avg=131072.0000 max=166795 max_rank=3 min=95947 min_rank=0 imbalance_ratio=1.272545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[95947, 149052, 148466, 166795, 107665, 147373, 105920, 127358] avg=131072.0000 max=166795 max_rank=3 min=95947 min_rank=0 imbalance_ratio=1.272545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[95947, 149052, 148466, 166795, 107665, 147373, 105920, 127358] avg=131072.0000 max=166795 max_rank=3 min=95947 min_rank=0 imbalance_ratio=1.272545 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=163840 expected_recv=256981 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[95947, 149052, 148466, 166795, 107665, 147373, 105920, 127358] avg=131072.0000 max=166795 max_rank=3 min=95947 min_rank=0 imbalance_ratio=1.272545 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=259712 slack_slots=67968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] avg=131072.0000 max=132384 max_rank=3 min=127358 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34411 max_load_reduction_pct=0.206307 moved_to_replicas=160679 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[95947, 149052, 148466, 166795, 107665, 147373, 105920, 127358] avg=131072.0000 max=166795 max_rank=3 min=95947 min_rank=0 imbalance_ratio=1.272545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] avg=131072.0000 max=132384 max_rank=3 min=127358 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34411 max_load_reduction_pct=0.206307 moved_to_replicas=160679 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] avg=131072.0000 max=132384 max_rank=3 min=127358 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34411 max_load_reduction_pct=0.206307 moved_to_replicas=160679 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] avg=131072.0000 max=132384 max_rank=3 min=127358 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34411 max_load_reduction_pct=0.206307 moved_to_replicas=160679 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[95947, 149052, 148466, 166795, 107665, 147373, 105920, 127358] avg=131072.0000 max=166795 max_rank=3 min=95947 min_rank=0 imbalance_ratio=1.272545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] avg=131072.0000 max=132384 max_rank=3 min=127358 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34411 max_load_reduction_pct=0.206307 moved_to_replicas=160679 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] avg=131072.0000 max=132384 max_rank=3 min=127358 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34411 max_load_reduction_pct=0.206307 moved_to_replicas=160679 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[139747, 193051, 144413, 154508, 128, 88611, 131766, 196352] avg=131072.0000 max=196352 max_rank=7 min=128 min_rank=4 imbalance_ratio=1.498047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[95947, 149052, 148466, 166795, 107665, 147373, 105920, 127358] avg=131072.0000 max=166795 max_rank=3 min=95947 min_rank=0 imbalance_ratio=1.272545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] avg=131072.0000 max=132384 max_rank=3 min=127358 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34411 max_load_reduction_pct=0.206307 moved_to_replicas=160679 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[139747, 193051, 144413, 154508, 128, 88611, 131766, 196352] avg=131072.0000 max=196352 max_rank=7 min=128 min_rank=4 imbalance_ratio=1.498047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[139747, 193051, 144413, 154508, 128, 88611, 131766, 196352] avg=131072.0000 max=196352 max_rank=7 min=128 min_rank=4 imbalance_ratio=1.498047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[139747, 193051, 144413, 154508, 128, 88611, 131766, 196352] avg=131072.0000 max=196352 max_rank=7 min=128 min_rank=4 imbalance_ratio=1.498047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[139747, 193051, 144413, 154508, 128, 88611, 131766, 196352] avg=131072.0000 max=196352 max_rank=7 min=128 min_rank=4 imbalance_ratio=1.498047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] avg=131072.0000 max=132384 max_rank=3 min=126349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63968 max_load_reduction_pct=0.325782 moved_to_replicas=330026 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[132383, 132382, 132383, 132384, 130967, 132384, 128335, 127358] avg=131072.0000 max=132384 max_rank=3 min=127358 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=34411 max_load_reduction_pct=0.206307 moved_to_replicas=160679 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[139747, 193051, 144413, 154508, 128, 88611, 131766, 196352] avg=131072.0000 max=196352 max_rank=7 min=128 min_rank=4 imbalance_ratio=1.498047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] avg=131072.0000 max=132384 max_rank=3 min=126349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63968 max_load_reduction_pct=0.325782 moved_to_replicas=330026 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] avg=131072.0000 max=132384 max_rank=3 min=126349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63968 max_load_reduction_pct=0.325782 moved_to_replicas=330026 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] avg=131072.0000 max=132384 max_rank=3 min=126349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63968 max_load_reduction_pct=0.325782 moved_to_replicas=330026 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[139747, 193051, 144413, 154508, 128, 88611, 131766, 196352] avg=131072.0000 max=196352 max_rank=7 min=128 min_rank=4 imbalance_ratio=1.498047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] avg=131072.0000 max=132384 max_rank=3 min=126349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63968 max_load_reduction_pct=0.325782 moved_to_replicas=330026 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] avg=131072.0000 max=132384 max_rank=3 min=126349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63968 max_load_reduction_pct=0.325782 moved_to_replicas=330026 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[226966, 110526, 161455, 70201, 65606, 196122, 98278, 119422] avg=131072.0000 max=226966 max_rank=0 min=65606 min_rank=4 imbalance_ratio=1.731613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[139747, 193051, 144413, 154508, 128, 88611, 131766, 196352] avg=131072.0000 max=196352 max_rank=7 min=128 min_rank=4 imbalance_ratio=1.498047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[226966, 110526, 161455, 70201, 65606, 196122, 98278, 119422] avg=131072.0000 max=226966 max_rank=0 min=65606 min_rank=4 imbalance_ratio=1.731613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] avg=131072.0000 max=132384 max_rank=3 min=126349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63968 max_load_reduction_pct=0.325782 moved_to_replicas=330026 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[226966, 110526, 161455, 70201, 65606, 196122, 98278, 119422] avg=131072.0000 max=226966 max_rank=0 min=65606 min_rank=4 imbalance_ratio=1.731613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[226966, 110526, 161455, 70201, 65606, 196122, 98278, 119422] avg=131072.0000 max=226966 max_rank=0 min=65606 min_rank=4 imbalance_ratio=1.731613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[226966, 110526, 161455, 70201, 65606, 196122, 98278, 119422] avg=131072.0000 max=226966 max_rank=0 min=65606 min_rank=4 imbalance_ratio=1.731613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] avg=131072.0000 max=132383 max_rank=0 min=127339 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=94583 max_load_reduction_pct=0.416728 moved_to_replicas=375762 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[132382, 132382, 132381, 132384, 126349, 128549, 131766, 132383] avg=131072.0000 max=132384 max_rank=3 min=126349 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63968 max_load_reduction_pct=0.325782 moved_to_replicas=330026 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[226966, 110526, 161455, 70201, 65606, 196122, 98278, 119422] avg=131072.0000 max=226966 max_rank=0 min=65606 min_rank=4 imbalance_ratio=1.731613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] avg=131072.0000 max=132383 max_rank=0 min=127339 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=94583 max_load_reduction_pct=0.416728 moved_to_replicas=375762 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] avg=131072.0000 max=132383 max_rank=0 min=127339 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=94583 max_load_reduction_pct=0.416728 moved_to_replicas=375762 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] avg=131072.0000 max=132383 max_rank=0 min=127339 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=94583 max_load_reduction_pct=0.416728 moved_to_replicas=375762 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] avg=131072.0000 max=132383 max_rank=0 min=127339 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=94583 max_load_reduction_pct=0.416728 moved_to_replicas=375762 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[226966, 110526, 161455, 70201, 65606, 196122, 98278, 119422] avg=131072.0000 max=226966 max_rank=0 min=65606 min_rank=4 imbalance_ratio=1.731613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] avg=131072.0000 max=132383 max_rank=0 min=127339 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=94583 max_load_reduction_pct=0.416728 moved_to_replicas=375762 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[92600, 98254, 98420, 196195, 159742, 196370, 47668, 159327] avg=131072.0000 max=196370 max_rank=5 min=47668 min_rank=6 imbalance_ratio=1.498184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[226966, 110526, 161455, 70201, 65606, 196122, 98278, 119422] avg=131072.0000 max=226966 max_rank=0 min=65606 min_rank=4 imbalance_ratio=1.731613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[92600, 98254, 98420, 196195, 159742, 196370, 47668, 159327] avg=131072.0000 max=196370 max_rank=5 min=47668 min_rank=6 imbalance_ratio=1.498184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[92600, 98254, 98420, 196195, 159742, 196370, 47668, 159327] avg=131072.0000 max=196370 max_rank=5 min=47668 min_rank=6 imbalance_ratio=1.498184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[92600, 98254, 98420, 196195, 159742, 196370, 47668, 159327] avg=131072.0000 max=196370 max_rank=5 min=47668 min_rank=6 imbalance_ratio=1.498184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] avg=131072.0000 max=132383 max_rank=0 min=127339 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=94583 max_load_reduction_pct=0.416728 moved_to_replicas=375762 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=125781 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63986 max_load_reduction_pct=0.325844 moved_to_replicas=367845 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[92600, 98254, 98420, 196195, 159742, 196370, 47668, 159327] avg=131072.0000 max=196370 max_rank=5 min=47668 min_rank=6 imbalance_ratio=1.498184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[132383, 132383, 132383, 132383, 131130, 132383, 127339, 128192] avg=131072.0000 max=132383 max_rank=0 min=127339 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=94583 max_load_reduction_pct=0.416728 moved_to_replicas=375762 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=125781 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63986 max_load_reduction_pct=0.325844 moved_to_replicas=367845 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[92600, 98254, 98420, 196195, 159742, 196370, 47668, 159327] avg=131072.0000 max=196370 max_rank=5 min=47668 min_rank=6 imbalance_ratio=1.498184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=125781 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63986 max_load_reduction_pct=0.325844 moved_to_replicas=367845 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=125781 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63986 max_load_reduction_pct=0.325844 moved_to_replicas=367845 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=125781 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63986 max_load_reduction_pct=0.325844 moved_to_replicas=367845 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[92600, 98254, 98420, 196195, 159742, 196370, 47668, 159327] avg=131072.0000 max=196370 max_rank=5 min=47668 min_rank=6 imbalance_ratio=1.498184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[65541, 65603, 196567, 197783, 163514, 65573, 131139, 162856] avg=131072.0000 max=197783 max_rank=3 min=65541 min_rank=0 imbalance_ratio=1.508965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=125781 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63986 max_load_reduction_pct=0.325844 moved_to_replicas=367845 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[92600, 98254, 98420, 196195, 159742, 196370, 47668, 159327] avg=131072.0000 max=196370 max_rank=5 min=47668 min_rank=6 imbalance_ratio=1.498184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[65541, 65603, 196567, 197783, 163514, 65573, 131139, 162856] avg=131072.0000 max=197783 max_rank=3 min=65541 min_rank=0 imbalance_ratio=1.508965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[65541, 65603, 196567, 197783, 163514, 65573, 131139, 162856] avg=131072.0000 max=197783 max_rank=3 min=65541 min_rank=0 imbalance_ratio=1.508965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[65541, 65603, 196567, 197783, 163514, 65573, 131139, 162856] avg=131072.0000 max=197783 max_rank=3 min=65541 min_rank=0 imbalance_ratio=1.508965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=125781 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63986 max_load_reduction_pct=0.325844 moved_to_replicas=367845 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] avg=131072.0000 max=132383 max_rank=2 min=128838 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65400 max_load_reduction_pct=0.330665 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[65541, 65603, 196567, 197783, 163514, 65573, 131139, 162856] avg=131072.0000 max=197783 max_rank=3 min=65541 min_rank=0 imbalance_ratio=1.508965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] avg=131072.0000 max=132383 max_rank=2 min=128838 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65400 max_load_reduction_pct=0.330665 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[131580, 129302, 125781, 132383, 132381, 132383, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=125781 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=63986 max_load_reduction_pct=0.325844 moved_to_replicas=367845 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] avg=131072.0000 max=132383 max_rank=2 min=128838 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65400 max_load_reduction_pct=0.330665 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[65541, 65603, 196567, 197783, 163514, 65573, 131139, 162856] avg=131072.0000 max=197783 max_rank=3 min=65541 min_rank=0 imbalance_ratio=1.508965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] avg=131072.0000 max=132383 max_rank=2 min=128838 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65400 max_load_reduction_pct=0.330665 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] avg=131072.0000 max=132383 max_rank=2 min=128838 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65400 max_load_reduction_pct=0.330665 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[65541, 65603, 196567, 197783, 163514, 65573, 131139, 162856] avg=131072.0000 max=197783 max_rank=3 min=65541 min_rank=0 imbalance_ratio=1.508965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[98914, 149630, 226806, 163808, 98706, 98679, 98136, 113897] avg=131072.0000 max=226806 max_rank=2 min=98136 min_rank=6 imbalance_ratio=1.730392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] avg=131072.0000 max=132383 max_rank=2 min=128838 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65400 max_load_reduction_pct=0.330665 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[98914, 149630, 226806, 163808, 98706, 98679, 98136, 113897] avg=131072.0000 max=226806 max_rank=2 min=98136 min_rank=6 imbalance_ratio=1.730392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[65541, 65603, 196567, 197783, 163514, 65573, 131139, 162856] avg=131072.0000 max=197783 max_rank=3 min=65541 min_rank=0 imbalance_ratio=1.508965 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[98914, 149630, 226806, 163808, 98706, 98679, 98136, 113897] avg=131072.0000 max=226806 max_rank=2 min=98136 min_rank=6 imbalance_ratio=1.730392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] avg=131072.0000 max=132383 max_rank=2 min=128838 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65400 max_load_reduction_pct=0.330665 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[98914, 149630, 226806, 163808, 98706, 98679, 98136, 113897] avg=131072.0000 max=226806 max_rank=2 min=98136 min_rank=6 imbalance_ratio=1.730392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] avg=131072.0000 max=132383 max_rank=2 min=127605 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94423 max_load_reduction_pct=0.416316 moved_to_replicas=287720 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[98914, 149630, 226806, 163808, 98706, 98679, 98136, 113897] avg=131072.0000 max=226806 max_rank=2 min=98136 min_rank=6 imbalance_ratio=1.730392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] avg=131072.0000 max=132383 max_rank=2 min=127605 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94423 max_load_reduction_pct=0.416316 moved_to_replicas=287720 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[98914, 149630, 226806, 163808, 98706, 98679, 98136, 113897] avg=131072.0000 max=226806 max_rank=2 min=98136 min_rank=6 imbalance_ratio=1.730392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[129434, 128838, 132383, 132383, 132383, 129633, 131139, 132383] avg=131072.0000 max=132383 max_rank=2 min=128838 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=65400 max_load_reduction_pct=0.330665 moved_to_replicas=382419 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] avg=131072.0000 max=132383 max_rank=2 min=127605 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94423 max_load_reduction_pct=0.416316 moved_to_replicas=287720 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] avg=131072.0000 max=132383 max_rank=2 min=127605 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94423 max_load_reduction_pct=0.416316 moved_to_replicas=287720 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] avg=131072.0000 max=132383 max_rank=2 min=127605 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94423 max_load_reduction_pct=0.416316 moved_to_replicas=287720 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[98914, 149630, 226806, 163808, 98706, 98679, 98136, 113897] avg=131072.0000 max=226806 max_rank=2 min=98136 min_rank=6 imbalance_ratio=1.730392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[134118, 65602, 177497, 130971, 180176, 131290, 98270, 130652] avg=131072.0000 max=180176 max_rank=4 min=65602 min_rank=1 imbalance_ratio=1.374634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] avg=131072.0000 max=132383 max_rank=2 min=127605 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94423 max_load_reduction_pct=0.416316 moved_to_replicas=287720 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[134118, 65602, 177497, 130971, 180176, 131290, 98270, 130652] avg=131072.0000 max=180176 max_rank=4 min=65602 min_rank=1 imbalance_ratio=1.374634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[134118, 65602, 177497, 130971, 180176, 131290, 98270, 130652] avg=131072.0000 max=180176 max_rank=4 min=65602 min_rank=1 imbalance_ratio=1.374634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[98914, 149630, 226806, 163808, 98706, 98679, 98136, 113897] avg=131072.0000 max=226806 max_rank=2 min=98136 min_rank=6 imbalance_ratio=1.730392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[134118, 65602, 177497, 130971, 180176, 131290, 98270, 130652] avg=131072.0000 max=180176 max_rank=4 min=65602 min_rank=1 imbalance_ratio=1.374634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] avg=131072.0000 max=132383 max_rank=2 min=127605 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94423 max_load_reduction_pct=0.416316 moved_to_replicas=287720 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] avg=131072.0000 max=132385 max_rank=2 min=127391 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=47791 max_load_reduction_pct=0.265246 moved_to_replicas=189368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] avg=131072.0000 max=132385 max_rank=2 min=127391 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=47791 max_load_reduction_pct=0.265246 moved_to_replicas=189368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[134118, 65602, 177497, 130971, 180176, 131290, 98270, 130652] avg=131072.0000 max=180176 max_rank=4 min=65602 min_rank=1 imbalance_ratio=1.374634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[134118, 65602, 177497, 130971, 180176, 131290, 98270, 130652] avg=131072.0000 max=180176 max_rank=4 min=65602 min_rank=1 imbalance_ratio=1.374634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] avg=131072.0000 max=132385 max_rank=2 min=127391 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=47791 max_load_reduction_pct=0.265246 moved_to_replicas=189368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[130339, 132381, 132383, 132383, 127605, 131441, 130898, 131146] avg=131072.0000 max=132383 max_rank=2 min=127605 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94423 max_load_reduction_pct=0.416316 moved_to_replicas=287720 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] avg=131072.0000 max=132385 max_rank=2 min=127391 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=47791 max_load_reduction_pct=0.265246 moved_to_replicas=189368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[134118, 65602, 177497, 130971, 180176, 131290, 98270, 130652] avg=131072.0000 max=180176 max_rank=4 min=65602 min_rank=1 imbalance_ratio=1.374634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] avg=131072.0000 max=132385 max_rank=2 min=127391 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=47791 max_load_reduction_pct=0.265246 moved_to_replicas=189368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[34017, 208583, 163823, 134677, 147886, 228510, 32807, 98273] avg=131072.0000 max=228510 max_rank=5 min=32807 min_rank=6 imbalance_ratio=1.743393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] avg=131072.0000 max=132385 max_rank=2 min=127391 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=47791 max_load_reduction_pct=0.265246 moved_to_replicas=189368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[34017, 208583, 163823, 134677, 147886, 228510, 32807, 98273] avg=131072.0000 max=228510 max_rank=5 min=32807 min_rank=6 imbalance_ratio=1.743393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[34017, 208583, 163823, 134677, 147886, 228510, 32807, 98273] avg=131072.0000 max=228510 max_rank=5 min=32807 min_rank=6 imbalance_ratio=1.743393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[134118, 65602, 177497, 130971, 180176, 131290, 98270, 130652] avg=131072.0000 max=180176 max_rank=4 min=65602 min_rank=1 imbalance_ratio=1.374634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[34017, 208583, 163823, 134677, 147886, 228510, 32807, 98273] avg=131072.0000 max=228510 max_rank=5 min=32807 min_rank=6 imbalance_ratio=1.743393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] avg=131072.0000 max=132385 max_rank=2 min=127391 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=47791 max_load_reduction_pct=0.265246 moved_to_replicas=189368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] avg=131072.0000 max=132384 max_rank=1 min=126747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96126 max_load_reduction_pct=0.420664 moved_to_replicas=442806 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] avg=131072.0000 max=132384 max_rank=1 min=126747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96126 max_load_reduction_pct=0.420664 moved_to_replicas=442806 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] avg=131072.0000 max=132384 max_rank=1 min=126747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96126 max_load_reduction_pct=0.420664 moved_to_replicas=442806 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[34017, 208583, 163823, 134677, 147886, 228510, 32807, 98273] avg=131072.0000 max=228510 max_rank=5 min=32807 min_rank=6 imbalance_ratio=1.743393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[34017, 208583, 163823, 134677, 147886, 228510, 32807, 98273] avg=131072.0000 max=228510 max_rank=5 min=32807 min_rank=6 imbalance_ratio=1.743393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[132383, 131121, 132385, 130971, 132383, 131290, 127391, 130652] avg=131072.0000 max=132385 max_rank=2 min=127391 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=47791 max_load_reduction_pct=0.265246 moved_to_replicas=189368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] avg=131072.0000 max=132384 max_rank=1 min=126747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96126 max_load_reduction_pct=0.420664 moved_to_replicas=442806 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[34017, 208583, 163823, 134677, 147886, 228510, 32807, 98273] avg=131072.0000 max=228510 max_rank=5 min=32807 min_rank=6 imbalance_ratio=1.743393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165424, 32906, 131047, 129323, 32806, 262042, 196719] avg=131072.0000 max=262042 max_rank=6 min=32806 min_rank=5 imbalance_ratio=1.999222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165424, 32906, 131047, 129323, 32806, 262042, 196719] avg=131072.0000 max=262042 max_rank=6 min=32806 min_rank=5 imbalance_ratio=1.999222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] avg=131072.0000 max=132384 max_rank=1 min=126747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96126 max_load_reduction_pct=0.420664 moved_to_replicas=442806 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] avg=131072.0000 max=132384 max_rank=1 min=126747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96126 max_load_reduction_pct=0.420664 moved_to_replicas=442806 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165424, 32906, 131047, 129323, 32806, 262042, 196719] avg=131072.0000 max=262042 max_rank=6 min=32806 min_rank=5 imbalance_ratio=1.999222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165424, 32906, 131047, 129323, 32806, 262042, 196719] avg=131072.0000 max=262042 max_rank=6 min=32806 min_rank=5 imbalance_ratio=1.999222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[34017, 208583, 163823, 134677, 147886, 228510, 32807, 98273] avg=131072.0000 max=228510 max_rank=5 min=32807 min_rank=6 imbalance_ratio=1.743393 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129602 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129659 max_load_reduction_pct=0.494802 moved_to_replicas=455134 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] avg=131072.0000 max=132384 max_rank=1 min=126747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96126 max_load_reduction_pct=0.420664 moved_to_replicas=442806 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129602 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129659 max_load_reduction_pct=0.494802 moved_to_replicas=455134 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129602 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129659 max_load_reduction_pct=0.494802 moved_to_replicas=455134 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165424, 32906, 131047, 129323, 32806, 262042, 196719] avg=131072.0000 max=262042 max_rank=6 min=32806 min_rank=5 imbalance_ratio=1.999222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129602 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129659 max_load_reduction_pct=0.494802 moved_to_replicas=455134 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165424, 32906, 131047, 129323, 32806, 262042, 196719] avg=131072.0000 max=262042 max_rank=6 min=32806 min_rank=5 imbalance_ratio=1.999222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[130978, 132384, 132383, 132383, 132384, 132383, 128934, 126747] avg=131072.0000 max=132384 max_rank=1 min=126747 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=96126 max_load_reduction_pct=0.420664 moved_to_replicas=442806 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[131060, 181000, 131074, 65401, 196550, 147735, 98540, 97216] avg=131072.0000 max=196550 max_rank=4 min=65401 min_rank=3 imbalance_ratio=1.499557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165424, 32906, 131047, 129323, 32806, 262042, 196719] avg=131072.0000 max=262042 max_rank=6 min=32806 min_rank=5 imbalance_ratio=1.999222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[131060, 181000, 131074, 65401, 196550, 147735, 98540, 97216] avg=131072.0000 max=196550 max_rank=4 min=65401 min_rank=3 imbalance_ratio=1.499557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129602 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129659 max_load_reduction_pct=0.494802 moved_to_replicas=455134 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[131060, 181000, 131074, 65401, 196550, 147735, 98540, 97216] avg=131072.0000 max=196550 max_rank=4 min=65401 min_rank=3 imbalance_ratio=1.499557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129602 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129659 max_load_reduction_pct=0.494802 moved_to_replicas=455134 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[131060, 181000, 131074, 65401, 196550, 147735, 98540, 97216] avg=131072.0000 max=196550 max_rank=4 min=65401 min_rank=3 imbalance_ratio=1.499557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[98309, 165424, 32906, 131047, 129323, 32806, 262042, 196719] avg=131072.0000 max=262042 max_rank=6 min=32806 min_rank=5 imbalance_ratio=1.999222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] avg=131072.0000 max=132385 max_rank=5 min=128623 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64165 max_load_reduction_pct=0.326456 moved_to_replicas=256437 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129602 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129659 max_load_reduction_pct=0.494802 moved_to_replicas=455134 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] avg=131072.0000 max=132385 max_rank=5 min=128623 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64165 max_load_reduction_pct=0.326456 moved_to_replicas=256437 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] avg=131072.0000 max=132385 max_rank=5 min=128623 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64165 max_load_reduction_pct=0.326456 moved_to_replicas=256437 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] avg=131072.0000 max=132385 max_rank=5 min=128623 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64165 max_load_reduction_pct=0.326456 moved_to_replicas=256437 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[131060, 181000, 131074, 65401, 196550, 147735, 98540, 97216] avg=131072.0000 max=196550 max_rank=4 min=65401 min_rank=3 imbalance_ratio=1.499557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[129884, 132383, 129806, 131047, 129602, 131088, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=129602 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=129659 max_load_reduction_pct=0.494802 moved_to_replicas=455134 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[131060, 181000, 131074, 65401, 196550, 147735, 98540, 97216] avg=131072.0000 max=196550 max_rank=4 min=65401 min_rank=3 imbalance_ratio=1.499557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[98374, 196481, 97245, 196556, 203874, 86399, 42811, 126836] avg=131072.0000 max=203874 max_rank=4 min=42811 min_rank=6 imbalance_ratio=1.555435 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[98374, 196481, 97245, 196556, 203874, 86399, 42811, 126836] avg=131072.0000 max=203874 max_rank=4 min=42811 min_rank=6 imbalance_ratio=1.555435 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[98374, 196481, 97245, 196556, 203874, 86399, 42811, 126836] avg=131072.0000 max=203874 max_rank=4 min=42811 min_rank=6 imbalance_ratio=1.555435 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[131060, 181000, 131074, 65401, 196550, 147735, 98540, 97216] avg=131072.0000 max=196550 max_rank=4 min=65401 min_rank=3 imbalance_ratio=1.499557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] avg=131072.0000 max=132385 max_rank=5 min=128623 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64165 max_load_reduction_pct=0.326456 moved_to_replicas=256437 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] avg=131072.0000 max=132385 max_rank=5 min=128623 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64165 max_load_reduction_pct=0.326456 moved_to_replicas=256437 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[98374, 196481, 97245, 196556, 203874, 86399, 42811, 126836] avg=131072.0000 max=203874 max_rank=4 min=42811 min_rank=6 imbalance_ratio=1.555435 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[131060, 181000, 131074, 65401, 196550, 147735, 98540, 97216] avg=131072.0000 max=196550 max_rank=4 min=65401 min_rank=3 imbalance_ratio=1.499557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] avg=131072.0000 max=132385 max_rank=5 min=126869 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71489 max_load_reduction_pct=0.350653 moved_to_replicas=398753 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] avg=131072.0000 max=132385 max_rank=5 min=126869 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71489 max_load_reduction_pct=0.350653 moved_to_replicas=398753 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] avg=131072.0000 max=132385 max_rank=5 min=126869 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71489 max_load_reduction_pct=0.350653 moved_to_replicas=398753 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] avg=131072.0000 max=132385 max_rank=5 min=128623 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64165 max_load_reduction_pct=0.326456 moved_to_replicas=256437 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] avg=131072.0000 max=132385 max_rank=5 min=126869 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71489 max_load_reduction_pct=0.350653 moved_to_replicas=398753 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[98374, 196481, 97245, 196556, 203874, 86399, 42811, 126836] avg=131072.0000 max=203874 max_rank=4 min=42811 min_rank=6 imbalance_ratio=1.555435 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[131060, 132383, 131074, 130921, 132383, 132385, 129747, 128623] avg=131072.0000 max=132385 max_rank=5 min=128623 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=64165 max_load_reduction_pct=0.326456 moved_to_replicas=256437 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[98374, 196481, 97245, 196556, 203874, 86399, 42811, 126836] avg=131072.0000 max=203874 max_rank=4 min=42811 min_rank=6 imbalance_ratio=1.555435 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[97035, 229362, 65154, 65669, 229403, 163894, 165237, 32822] avg=131072.0000 max=229403 max_rank=4 min=32822 min_rank=7 imbalance_ratio=1.750206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[97035, 229362, 65154, 65669, 229403, 163894, 165237, 32822] avg=131072.0000 max=229403 max_rank=4 min=32822 min_rank=7 imbalance_ratio=1.750206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[97035, 229362, 65154, 65669, 229403, 163894, 165237, 32822] avg=131072.0000 max=229403 max_rank=4 min=32822 min_rank=7 imbalance_ratio=1.750206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[98374, 196481, 97245, 196556, 203874, 86399, 42811, 126836] avg=131072.0000 max=203874 max_rank=4 min=42811 min_rank=6 imbalance_ratio=1.555435 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] avg=131072.0000 max=132385 max_rank=5 min=126869 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71489 max_load_reduction_pct=0.350653 moved_to_replicas=398753 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] avg=131072.0000 max=132385 max_rank=5 min=126869 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71489 max_load_reduction_pct=0.350653 moved_to_replicas=398753 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[97035, 229362, 65154, 65669, 229403, 163894, 165237, 32822] avg=131072.0000 max=229403 max_rank=4 min=32822 min_rank=7 imbalance_ratio=1.750206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] avg=131072.0000 max=132383 max_rank=1 min=128543 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97020 max_load_reduction_pct=0.422924 moved_to_replicas=518345 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[98374, 196481, 97245, 196556, 203874, 86399, 42811, 126836] avg=131072.0000 max=203874 max_rank=4 min=42811 min_rank=6 imbalance_ratio=1.555435 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] avg=131072.0000 max=132383 max_rank=1 min=128543 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97020 max_load_reduction_pct=0.422924 moved_to_replicas=518345 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] avg=131072.0000 max=132383 max_rank=1 min=128543 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97020 max_load_reduction_pct=0.422924 moved_to_replicas=518345 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] avg=131072.0000 max=132385 max_rank=5 min=126869 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71489 max_load_reduction_pct=0.350653 moved_to_replicas=398753 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] avg=131072.0000 max=132383 max_rank=1 min=128543 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97020 max_load_reduction_pct=0.422924 moved_to_replicas=518345 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[131134, 132383, 128658, 132383, 132382, 132385, 132382, 126869] avg=131072.0000 max=132385 max_rank=5 min=126869 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71489 max_load_reduction_pct=0.350653 moved_to_replicas=398753 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[97035, 229362, 65154, 65669, 229403, 163894, 165237, 32822] avg=131072.0000 max=229403 max_rank=4 min=32822 min_rank=7 imbalance_ratio=1.750206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[97035, 229362, 65154, 65669, 229403, 163894, 165237, 32822] avg=131072.0000 max=229403 max_rank=4 min=32822 min_rank=7 imbalance_ratio=1.750206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[131400, 163888, 98369, 98333, 196565, 98564, 163166, 98291] avg=131072.0000 max=196565 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[131400, 163888, 98369, 98333, 196565, 98564, 163166, 98291] avg=131072.0000 max=196565 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[131400, 163888, 98369, 98333, 196565, 98564, 163166, 98291] avg=131072.0000 max=196565 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[97035, 229362, 65154, 65669, 229403, 163894, 165237, 32822] avg=131072.0000 max=229403 max_rank=4 min=32822 min_rank=7 imbalance_ratio=1.750206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] avg=131072.0000 max=132383 max_rank=1 min=128543 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97020 max_load_reduction_pct=0.422924 moved_to_replicas=518345 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[131400, 163888, 98369, 98333, 196565, 98564, 163166, 98291] avg=131072.0000 max=196565 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] avg=131072.0000 max=132383 max_rank=1 min=128543 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97020 max_load_reduction_pct=0.422924 moved_to_replicas=518345 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64182 max_load_reduction_pct=0.326518 moved_to_replicas=252030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[97035, 229362, 65154, 65669, 229403, 163894, 165237, 32822] avg=131072.0000 max=229403 max_rank=4 min=32822 min_rank=7 imbalance_ratio=1.750206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64182 max_load_reduction_pct=0.326518 moved_to_replicas=252030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64182 max_load_reduction_pct=0.326518 moved_to_replicas=252030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] avg=131072.0000 max=132383 max_rank=1 min=128543 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97020 max_load_reduction_pct=0.422924 moved_to_replicas=518345 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64182 max_load_reduction_pct=0.326518 moved_to_replicas=252030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[131400, 163888, 98369, 98333, 196565, 98564, 163166, 98291] avg=131072.0000 max=196565 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[129793, 132383, 130668, 130040, 132383, 132383, 132383, 128543] avg=131072.0000 max=132383 max_rank=1 min=128543 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=97020 max_load_reduction_pct=0.422924 moved_to_replicas=518345 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[131400, 163888, 98369, 98333, 196565, 98564, 163166, 98291] avg=131072.0000 max=196565 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[96012, 102146, 163962, 194570, 65638, 165576, 65628, 195044] avg=131072.0000 max=195044 max_rank=7 min=65628 min_rank=6 imbalance_ratio=1.488068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[96012, 102146, 163962, 194570, 65638, 165576, 65628, 195044] avg=131072.0000 max=195044 max_rank=7 min=65628 min_rank=6 imbalance_ratio=1.488068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[96012, 102146, 163962, 194570, 65638, 165576, 65628, 195044] avg=131072.0000 max=195044 max_rank=7 min=65628 min_rank=6 imbalance_ratio=1.488068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[131400, 163888, 98369, 98333, 196565, 98564, 163166, 98291] avg=131072.0000 max=196565 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64182 max_load_reduction_pct=0.326518 moved_to_replicas=252030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[96012, 102146, 163962, 194570, 65638, 165576, 65628, 195044] avg=131072.0000 max=195044 max_rank=7 min=65628 min_rank=6 imbalance_ratio=1.488068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64182 max_load_reduction_pct=0.326518 moved_to_replicas=252030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] avg=131072.0000 max=132383 max_rank=1 min=127219 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62661 max_load_reduction_pct=0.321266 moved_to_replicas=379213 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] avg=131072.0000 max=132383 max_rank=1 min=127219 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62661 max_load_reduction_pct=0.321266 moved_to_replicas=379213 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[131400, 163888, 98369, 98333, 196565, 98564, 163166, 98291] avg=131072.0000 max=196565 max_rank=4 min=98291 min_rank=7 imbalance_ratio=1.499672 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] avg=131072.0000 max=132383 max_rank=1 min=127219 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62661 max_load_reduction_pct=0.321266 moved_to_replicas=379213 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64182 max_load_reduction_pct=0.326518 moved_to_replicas=252030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] avg=131072.0000 max=132383 max_rank=1 min=127219 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62661 max_load_reduction_pct=0.321266 moved_to_replicas=379213 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[96012, 102146, 163962, 194570, 65638, 165576, 65628, 195044] avg=131072.0000 max=195044 max_rank=7 min=65628 min_rank=6 imbalance_ratio=1.488068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[96012, 102146, 163962, 194570, 65638, 165576, 65628, 195044] avg=131072.0000 max=195044 max_rank=7 min=65628 min_rank=6 imbalance_ratio=1.488068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[131400, 132383, 129874, 129755, 132383, 129347, 132383, 131051] avg=131072.0000 max=132383 max_rank=1 min=129347 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64182 max_load_reduction_pct=0.326518 moved_to_replicas=252030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[229300, 196617, 71, 131059, 98298, 131097, 131106, 131028] avg=131072.0000 max=229300 max_rank=0 min=71 min_rank=2 imbalance_ratio=1.749420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[229300, 196617, 71, 131059, 98298, 131097, 131106, 131028] avg=131072.0000 max=229300 max_rank=0 min=71 min_rank=2 imbalance_ratio=1.749420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[229300, 196617, 71, 131059, 98298, 131097, 131106, 131028] avg=131072.0000 max=229300 max_rank=0 min=71 min_rank=2 imbalance_ratio=1.749420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] avg=131072.0000 max=132383 max_rank=1 min=127219 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62661 max_load_reduction_pct=0.321266 moved_to_replicas=379213 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[96012, 102146, 163962, 194570, 65638, 165576, 65628, 195044] avg=131072.0000 max=195044 max_rank=7 min=65628 min_rank=6 imbalance_ratio=1.488068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[229300, 196617, 71, 131059, 98298, 131097, 131106, 131028] avg=131072.0000 max=229300 max_rank=0 min=71 min_rank=2 imbalance_ratio=1.749420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] avg=131072.0000 max=132383 max_rank=1 min=127219 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62661 max_load_reduction_pct=0.321266 moved_to_replicas=379213 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96917 max_load_reduction_pct=0.422665 moved_to_replicas=322294 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96917 max_load_reduction_pct=0.422665 moved_to_replicas=322294 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[96012, 102146, 163962, 194570, 65638, 165576, 65628, 195044] avg=131072.0000 max=195044 max_rank=7 min=65628 min_rank=6 imbalance_ratio=1.488068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96917 max_load_reduction_pct=0.422665 moved_to_replicas=322294 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] avg=131072.0000 max=132383 max_rank=1 min=127219 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62661 max_load_reduction_pct=0.321266 moved_to_replicas=379213 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96917 max_load_reduction_pct=0.422665 moved_to_replicas=322294 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[229300, 196617, 71, 131059, 98298, 131097, 131106, 131028] avg=131072.0000 max=229300 max_rank=0 min=71 min_rank=2 imbalance_ratio=1.749420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[229300, 196617, 71, 131059, 98298, 131097, 131106, 131028] avg=131072.0000 max=229300 max_rank=0 min=71 min_rank=2 imbalance_ratio=1.749420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[127219, 132383, 132383, 132383, 128297, 132383, 131145, 132383] avg=131072.0000 max=132383 max_rank=1 min=127219 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=62661 max_load_reduction_pct=0.321266 moved_to_replicas=379213 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[196360, 196609, 32838, 163765, 32821, 163771, 65592, 196820] avg=131072.0000 max=196820 max_rank=7 min=32821 min_rank=4 imbalance_ratio=1.501617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[196360, 196609, 32838, 163765, 32821, 163771, 65592, 196820] avg=131072.0000 max=196820 max_rank=7 min=32821 min_rank=4 imbalance_ratio=1.501617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[196360, 196609, 32838, 163765, 32821, 163771, 65592, 196820] avg=131072.0000 max=196820 max_rank=7 min=32821 min_rank=4 imbalance_ratio=1.501617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96917 max_load_reduction_pct=0.422665 moved_to_replicas=322294 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[229300, 196617, 71, 131059, 98298, 131097, 131106, 131028] avg=131072.0000 max=229300 max_rank=0 min=71 min_rank=2 imbalance_ratio=1.749420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[196360, 196609, 32838, 163765, 32821, 163771, 65592, 196820] avg=131072.0000 max=196820 max_rank=7 min=32821 min_rank=4 imbalance_ratio=1.501617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96917 max_load_reduction_pct=0.422665 moved_to_replicas=322294 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] avg=131072.0000 max=132383 max_rank=0 min=128266 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64437 max_load_reduction_pct=0.327391 moved_to_replicas=510644 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] avg=131072.0000 max=132383 max_rank=0 min=128266 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64437 max_load_reduction_pct=0.327391 moved_to_replicas=510644 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] avg=131072.0000 max=132383 max_rank=0 min=128266 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64437 max_load_reduction_pct=0.327391 moved_to_replicas=510644 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[229300, 196617, 71, 131059, 98298, 131097, 131106, 131028] avg=131072.0000 max=229300 max_rank=0 min=71 min_rank=2 imbalance_ratio=1.749420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96917 max_load_reduction_pct=0.422665 moved_to_replicas=322294 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] avg=131072.0000 max=132383 max_rank=0 min=128266 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64437 max_load_reduction_pct=0.327391 moved_to_replicas=510644 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[196360, 196609, 32838, 163765, 32821, 163771, 65592, 196820] avg=131072.0000 max=196820 max_rank=7 min=32821 min_rank=4 imbalance_ratio=1.501617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[196360, 196609, 32838, 163765, 32821, 163771, 65592, 196820] avg=131072.0000 max=196820 max_rank=7 min=32821 min_rank=4 imbalance_ratio=1.501617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[132383, 132383, 129747, 131059, 129773, 131097, 131106, 131028] avg=131072.0000 max=132383 max_rank=0 min=129747 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96917 max_load_reduction_pct=0.422665 moved_to_replicas=322294 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] avg=131072.0000 max=132383 max_rank=0 min=128266 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64437 max_load_reduction_pct=0.327391 moved_to_replicas=510644 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] avg=131072.0000 max=132383 max_rank=0 min=128266 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64437 max_load_reduction_pct=0.327391 moved_to_replicas=510644 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[196360, 196609, 32838, 163765, 32821, 163771, 65592, 196820] avg=131072.0000 max=196820 max_rank=7 min=32821 min_rank=4 imbalance_ratio=1.501617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[196360, 196609, 32838, 163765, 32821, 163771, 65592, 196820] avg=131072.0000 max=196820 max_rank=7 min=32821 min_rank=4 imbalance_ratio=1.501617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] avg=131072.0000 max=132383 max_rank=0 min=128266 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64437 max_load_reduction_pct=0.327391 moved_to_replicas=510644 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[132383, 132383, 128662, 132383, 128266, 132383, 129733, 132383] avg=131072.0000 max=132383 max_rank=0 min=128266 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64437 max_load_reduction_pct=0.327391 moved_to_replicas=510644 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=4194473 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1010048 peak_demand=132383 required_capacity=1.010002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=4201878 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1003520 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=4213148 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=990720 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=4187455 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016832 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=4172823 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1032320 peak_demand=132383 required_capacity=1.010002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=4221057 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=983808 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=4175276 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1027456 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=4188322 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1013248 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:17:55] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 573.9 | throughput per GPU (TFLOP/s/GPU): 580.9 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.901830E+00 | loss scale: 1.0 | grad norm: 6.193 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=163840 expected_recv=258501 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=163840 expected_recv=258834 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=163840 expected_recv=258557 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262193 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=163840 expected_recv=258557 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=163840 expected_recv=258501 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=163840 expected_recv=258834 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=260608 slack_slots=67072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262193 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=163840 expected_recv=264337 expected_overflow=0 exact_recv_per_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=163840 expected_recv=258199 expected_overflow=0 exact_recv_per_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=163840 expected_recv=261944 expected_overflow=0 exact_recv_per_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=163840 expected_recv=264337 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262272 expected_overflow=0 exact_recv_per_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=163840 expected_recv=258199 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=163840 expected_recv=261944 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262272 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=163840 expected_recv=256101 expected_overflow=0 exact_recv_per_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=261120 slack_slots=66560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=163840 expected_recv=256101 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=259200 slack_slots=68480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=163840 expected_recv=260903 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=163840 expected_recv=260903 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=163840 expected_recv=254121 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=163840 expected_recv=258298 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=163840 expected_recv=254121 actual_valid_recv=257024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=163840 expected_recv=258298 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=257024 slack_slots=70656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=260736 slack_slots=66944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=163840 expected_recv=258949 expected_overflow=0 exact_recv_per_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=163840 expected_recv=258949 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262063 expected_overflow=0 exact_recv_per_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=163840 expected_recv=254231 expected_overflow=0 exact_recv_per_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262845 expected_overflow=0 exact_recv_per_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262063 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=163840 expected_recv=259793 expected_overflow=0 exact_recv_per_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=163840 expected_recv=254231 actual_valid_recv=257280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262845 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=257280 slack_slots=70400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=163840 expected_recv=259793 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262073 expected_overflow=0 exact_recv_per_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=163840 expected_recv=254695 expected_overflow=0 exact_recv_per_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262073 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=163840 expected_recv=254695 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=163840 expected_recv=261528 expected_overflow=0 exact_recv_per_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=257408 slack_slots=70272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=163840 expected_recv=261528 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261955 expected_overflow=0 exact_recv_per_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261768 expected_overflow=0 exact_recv_per_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261955 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261768 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=163840 expected_recv=252386 expected_overflow=0 exact_recv_per_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261978 expected_overflow=0 exact_recv_per_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=163840 expected_recv=252386 actual_valid_recv=254336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261978 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=254336 slack_slots=73344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=163840 expected_recv=259862 expected_overflow=0 exact_recv_per_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262593 expected_overflow=0 exact_recv_per_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=163840 expected_recv=259862 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=163840 expected_recv=260234 expected_overflow=0 exact_recv_per_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262593 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=163840 expected_recv=260234 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=163840 expected_recv=258001 expected_overflow=0 exact_recv_per_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262165 expected_overflow=0 exact_recv_per_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262144 expected_overflow=0 exact_recv_per_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=163840 expected_recv=258001 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262165 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262144 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=260480 slack_slots=67200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=163840 expected_recv=264362 expected_overflow=0 exact_recv_per_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=163840 expected_recv=264362 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=163840 expected_recv=255371 expected_overflow=0 exact_recv_per_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=163840 expected_recv=255371 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=163840 expected_recv=259547 expected_overflow=0 exact_recv_per_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=257792 slack_slots=69888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=163840 expected_recv=256212 expected_overflow=0 exact_recv_per_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=163840 expected_recv=259547 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=163840 expected_recv=261956 expected_overflow=0 exact_recv_per_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=163840 expected_recv=256212 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=163840 expected_recv=261956 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=259072 slack_slots=68608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=163840 expected_recv=258803 expected_overflow=0 exact_recv_per_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=163840 expected_recv=260405 expected_overflow=0 exact_recv_per_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=163840 expected_recv=258803 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=163840 expected_recv=260405 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=261760 slack_slots=65920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262118 expected_overflow=0 exact_recv_per_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262142 expected_overflow=0 exact_recv_per_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262118 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262769 expected_overflow=0 exact_recv_per_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262142 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262769 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=163840 expected_recv=256292 expected_overflow=0 exact_recv_per_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=163840 expected_recv=256292 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=258304 slack_slots=69376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262331 expected_overflow=0 exact_recv_per_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=163840 expected_recv=259343 expected_overflow=0 exact_recv_per_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262331 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=163840 expected_recv=259343 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=163840 expected_recv=256982 expected_overflow=0 exact_recv_per_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=163840 expected_recv=258447 expected_overflow=0 exact_recv_per_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=163840 expected_recv=256982 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=163840 expected_recv=258447 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=259456 slack_slots=68224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262155 expected_overflow=0 exact_recv_per_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=163840 expected_recv=261991 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262155 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=163840 expected_recv=261757 expected_overflow=0 exact_recv_per_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=163840 expected_recv=263514 expected_overflow=0 exact_recv_per_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=163840 expected_recv=261991 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=163840 expected_recv=264211 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=163840 expected_recv=261757 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=163840 expected_recv=263514 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=163840 expected_recv=263906 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=163840 expected_recv=264211 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=163840 expected_recv=247980 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=163840 expected_recv=263906 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=163840 expected_recv=258399 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=163840 expected_recv=247980 actual_valid_recv=250112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=250112 slack_slots=77568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=163840 expected_recv=258399 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=163840 expected_recv=260338 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=260736 slack_slots=66944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=163840 expected_recv=260338 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=163840 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=163840 expected_recv=261699 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=163840 expected_recv=264763 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=268032 slack_slots=59648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=163840 expected_recv=261699 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=163840 expected_recv=260103 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=163840 expected_recv=260103 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=163840 expected_recv=262317 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=163840 expected_recv=260174 expected_overflow=0 exact_recv_per_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=163840 expected_recv=261842 expected_overflow=0 exact_recv_per_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=163840 expected_recv=262317 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=163840 expected_recv=260174 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=163840 expected_recv=261842 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=163840 expected_recv=255596 expected_overflow=0 exact_recv_per_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=163840 expected_recv=259927 expected_overflow=0 exact_recv_per_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=163840 expected_recv=255596 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=163840 expected_recv=259927 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=257920 slack_slots=69760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262999 expected_overflow=0 exact_recv_per_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262999 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262541 expected_overflow=0 exact_recv_per_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262244 expected_overflow=0 exact_recv_per_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262541 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262244 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261853 expected_overflow=0 exact_recv_per_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261853 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=163840 expected_recv=259775 expected_overflow=0 exact_recv_per_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=163840 expected_recv=250854 expected_overflow=0 exact_recv_per_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261673 expected_overflow=0 exact_recv_per_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=163840 expected_recv=259775 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[146966, 155471, 118370, 126119, 141408, 93777, 114864, 151601] avg=131072.0000 max=155471 max_rank=1 min=93777 min_rank=5 imbalance_ratio=1.186150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=163840 expected_recv=250854 actual_valid_recv=253184 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[146966, 155471, 118370, 126119, 141408, 93777, 114864, 151601] avg=131072.0000 max=155471 max_rank=1 min=93777 min_rank=5 imbalance_ratio=1.186150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261673 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=253184 slack_slots=74496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23087 max_load_reduction_pct=0.148497 moved_to_replicas=128075 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23087 max_load_reduction_pct=0.148497 moved_to_replicas=128075 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=163840 expected_recv=254787 expected_overflow=0 exact_recv_per_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=163840 expected_recv=254787 actual_valid_recv=257152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[146966, 155471, 118370, 126119, 141408, 93777, 114864, 151601] avg=131072.0000 max=155471 max_rank=1 min=93777 min_rank=5 imbalance_ratio=1.186150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[146966, 155471, 118370, 126119, 141408, 93777, 114864, 151601] avg=131072.0000 max=155471 max_rank=1 min=93777 min_rank=5 imbalance_ratio=1.186150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[146966, 155471, 118370, 126119, 141408, 93777, 114864, 151601] avg=131072.0000 max=155471 max_rank=1 min=93777 min_rank=5 imbalance_ratio=1.186150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=257152 slack_slots=70528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[146966, 155471, 118370, 126119, 141408, 93777, 114864, 151601] avg=131072.0000 max=155471 max_rank=1 min=93777 min_rank=5 imbalance_ratio=1.186150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[97308, 247458, 130998, 115906, 56754, 131954, 151700, 116498] avg=131072.0000 max=247458 max_rank=1 min=56754 min_rank=4 imbalance_ratio=1.887955 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[97308, 247458, 130998, 115906, 56754, 131954, 151700, 116498] avg=131072.0000 max=247458 max_rank=1 min=56754 min_rank=4 imbalance_ratio=1.887955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23087 max_load_reduction_pct=0.148497 moved_to_replicas=128075 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23087 max_load_reduction_pct=0.148497 moved_to_replicas=128075 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23087 max_load_reduction_pct=0.148497 moved_to_replicas=128075 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23087 max_load_reduction_pct=0.148497 moved_to_replicas=128075 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=126052 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=115074 max_load_reduction_pct=0.465024 moved_to_replicas=268707 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=126052 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=115074 max_load_reduction_pct=0.465024 moved_to_replicas=268707 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[146966, 155471, 118370, 126119, 141408, 93777, 114864, 151601] avg=131072.0000 max=155471 max_rank=1 min=93777 min_rank=5 imbalance_ratio=1.186150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[146966, 155471, 118370, 126119, 141408, 93777, 114864, 151601] avg=131072.0000 max=155471 max_rank=1 min=93777 min_rank=5 imbalance_ratio=1.186150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[97308, 247458, 130998, 115906, 56754, 131954, 151700, 116498] avg=131072.0000 max=247458 max_rank=1 min=56754 min_rank=4 imbalance_ratio=1.887955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[97308, 247458, 130998, 115906, 56754, 131954, 151700, 116498] avg=131072.0000 max=247458 max_rank=1 min=56754 min_rank=4 imbalance_ratio=1.887955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[97308, 247458, 130998, 115906, 56754, 131954, 151700, 116498] avg=131072.0000 max=247458 max_rank=1 min=56754 min_rank=4 imbalance_ratio=1.887955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[133413, 247180, 193220, 114034, 98689, 130753, 88106, 43181] avg=131072.0000 max=247180 max_rank=1 min=43181 min_rank=7 imbalance_ratio=1.885834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[97308, 247458, 130998, 115906, 56754, 131954, 151700, 116498] avg=131072.0000 max=247458 max_rank=1 min=56754 min_rank=4 imbalance_ratio=1.887955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23087 max_load_reduction_pct=0.148497 moved_to_replicas=128075 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[133413, 247180, 193220, 114034, 98689, 130753, 88106, 43181] avg=131072.0000 max=247180 max_rank=1 min=43181 min_rank=7 imbalance_ratio=1.885834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[132384, 132383, 132382, 126119, 132384, 128158, 132383, 132383] avg=131072.0000 max=132384 max_rank=0 min=126119 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=23087 max_load_reduction_pct=0.148497 moved_to_replicas=128075 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=126052 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=115074 max_load_reduction_pct=0.465024 moved_to_replicas=268707 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=126052 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=115074 max_load_reduction_pct=0.465024 moved_to_replicas=268707 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=126052 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=115074 max_load_reduction_pct=0.465024 moved_to_replicas=268707 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] avg=131072.0000 max=132385 max_rank=7 min=126301 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=114795 max_load_reduction_pct=0.464419 moved_to_replicas=356454 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=126052 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=115074 max_load_reduction_pct=0.465024 moved_to_replicas=268707 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] avg=131072.0000 max=132385 max_rank=7 min=126301 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=114795 max_load_reduction_pct=0.464419 moved_to_replicas=356454 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[97308, 247458, 130998, 115906, 56754, 131954, 151700, 116498] avg=131072.0000 max=247458 max_rank=1 min=56754 min_rank=4 imbalance_ratio=1.887955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[133413, 247180, 193220, 114034, 98689, 130753, 88106, 43181] avg=131072.0000 max=247180 max_rank=1 min=43181 min_rank=7 imbalance_ratio=1.885834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[97308, 247458, 130998, 115906, 56754, 131954, 151700, 116498] avg=131072.0000 max=247458 max_rank=1 min=56754 min_rank=4 imbalance_ratio=1.887955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[133413, 247180, 193220, 114034, 98689, 130753, 88106, 43181] avg=131072.0000 max=247180 max_rank=1 min=43181 min_rank=7 imbalance_ratio=1.885834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[133413, 247180, 193220, 114034, 98689, 130753, 88106, 43181] avg=131072.0000 max=247180 max_rank=1 min=43181 min_rank=7 imbalance_ratio=1.885834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[119364, 78974, 222005, 169036, 99883, 195858, 65176, 98280] avg=131072.0000 max=222005 max_rank=2 min=65176 min_rank=6 imbalance_ratio=1.693764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[133413, 247180, 193220, 114034, 98689, 130753, 88106, 43181] avg=131072.0000 max=247180 max_rank=1 min=43181 min_rank=7 imbalance_ratio=1.885834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[119364, 78974, 222005, 169036, 99883, 195858, 65176, 98280] avg=131072.0000 max=222005 max_rank=2 min=65176 min_rank=6 imbalance_ratio=1.693764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=126052 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=115074 max_load_reduction_pct=0.465024 moved_to_replicas=268707 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] avg=131072.0000 max=132385 max_rank=7 min=126301 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=114795 max_load_reduction_pct=0.464419 moved_to_replicas=356454 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] avg=131072.0000 max=132385 max_rank=7 min=126301 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=114795 max_load_reduction_pct=0.464419 moved_to_replicas=356454 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[130040, 132383, 130998, 132383, 126052, 131954, 132384, 132382] avg=131072.0000 max=132384 max_rank=6 min=126052 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=115074 max_load_reduction_pct=0.465024 moved_to_replicas=268707 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] avg=131072.0000 max=132384 max_rank=2 min=125934 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=89621 max_load_reduction_pct=0.403689 moved_to_replicas=383071 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] avg=131072.0000 max=132385 max_rank=7 min=126301 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=114795 max_load_reduction_pct=0.464419 moved_to_replicas=356454 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] avg=131072.0000 max=132385 max_rank=7 min=126301 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=114795 max_load_reduction_pct=0.464419 moved_to_replicas=356454 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] avg=131072.0000 max=132384 max_rank=2 min=125934 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=89621 max_load_reduction_pct=0.403689 moved_to_replicas=383071 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[133413, 247180, 193220, 114034, 98689, 130753, 88106, 43181] avg=131072.0000 max=247180 max_rank=1 min=43181 min_rank=7 imbalance_ratio=1.885834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[119364, 78974, 222005, 169036, 99883, 195858, 65176, 98280] avg=131072.0000 max=222005 max_rank=2 min=65176 min_rank=6 imbalance_ratio=1.693764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[119364, 78974, 222005, 169036, 99883, 195858, 65176, 98280] avg=131072.0000 max=222005 max_rank=2 min=65176 min_rank=6 imbalance_ratio=1.693764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[31627, 98284, 112871, 196596, 215605, 98445, 130799, 164349] avg=131072.0000 max=215605 max_rank=4 min=31627 min_rank=0 imbalance_ratio=1.644936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[119364, 78974, 222005, 169036, 99883, 195858, 65176, 98280] avg=131072.0000 max=222005 max_rank=2 min=65176 min_rank=6 imbalance_ratio=1.693764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[133413, 247180, 193220, 114034, 98689, 130753, 88106, 43181] avg=131072.0000 max=247180 max_rank=1 min=43181 min_rank=7 imbalance_ratio=1.885834 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[31627, 98284, 112871, 196596, 215605, 98445, 130799, 164349] avg=131072.0000 max=215605 max_rank=4 min=31627 min_rank=0 imbalance_ratio=1.644936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[119364, 78974, 222005, 169036, 99883, 195858, 65176, 98280] avg=131072.0000 max=222005 max_rank=2 min=65176 min_rank=6 imbalance_ratio=1.693764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] avg=131072.0000 max=132384 max_rank=2 min=125934 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=89621 max_load_reduction_pct=0.403689 moved_to_replicas=383071 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] avg=131072.0000 max=132385 max_rank=7 min=126301 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=114795 max_load_reduction_pct=0.464419 moved_to_replicas=356454 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] avg=131072.0000 max=132383 max_rank=3 min=127305 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=83222 max_load_reduction_pct=0.385993 moved_to_replicas=365524 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] avg=131072.0000 max=132384 max_rank=2 min=125934 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=89621 max_load_reduction_pct=0.403689 moved_to_replicas=383071 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] avg=131072.0000 max=132384 max_rank=2 min=125934 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=89621 max_load_reduction_pct=0.403689 moved_to_replicas=383071 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[132383, 132382, 132380, 132383, 129609, 130753, 126301, 132385] avg=131072.0000 max=132385 max_rank=7 min=126301 min_rank=6 imbalance_ratio=1.010017 max_load_reduction=114795 max_load_reduction_pct=0.464419 moved_to_replicas=356454 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] avg=131072.0000 max=132383 max_rank=3 min=127305 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=83222 max_load_reduction_pct=0.385993 moved_to_replicas=365524 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] avg=131072.0000 max=132384 max_rank=2 min=125934 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=89621 max_load_reduction_pct=0.403689 moved_to_replicas=383071 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[31627, 98284, 112871, 196596, 215605, 98445, 130799, 164349] avg=131072.0000 max=215605 max_rank=4 min=31627 min_rank=0 imbalance_ratio=1.644936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[119364, 78974, 222005, 169036, 99883, 195858, 65176, 98280] avg=131072.0000 max=222005 max_rank=2 min=65176 min_rank=6 imbalance_ratio=1.693764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[67204, 130984, 218467, 140179, 66403, 163829, 98249, 163261] avg=131072.0000 max=218467 max_rank=2 min=66403 min_rank=4 imbalance_ratio=1.666771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[31627, 98284, 112871, 196596, 215605, 98445, 130799, 164349] avg=131072.0000 max=215605 max_rank=4 min=31627 min_rank=0 imbalance_ratio=1.644936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[31627, 98284, 112871, 196596, 215605, 98445, 130799, 164349] avg=131072.0000 max=215605 max_rank=4 min=31627 min_rank=0 imbalance_ratio=1.644936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[67204, 130984, 218467, 140179, 66403, 163829, 98249, 163261] avg=131072.0000 max=218467 max_rank=2 min=66403 min_rank=4 imbalance_ratio=1.666771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] avg=131072.0000 max=132383 max_rank=3 min=127305 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=83222 max_load_reduction_pct=0.385993 moved_to_replicas=365524 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[119364, 78974, 222005, 169036, 99883, 195858, 65176, 98280] avg=131072.0000 max=222005 max_rank=2 min=65176 min_rank=6 imbalance_ratio=1.693764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[31627, 98284, 112871, 196596, 215605, 98445, 130799, 164349] avg=131072.0000 max=215605 max_rank=4 min=31627 min_rank=0 imbalance_ratio=1.644936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] avg=131072.0000 max=132384 max_rank=2 min=125934 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=89621 max_load_reduction_pct=0.403689 moved_to_replicas=383071 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] avg=131072.0000 max=132384 max_rank=2 min=126616 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86083 max_load_reduction_pct=0.394032 moved_to_replicas=311922 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] avg=131072.0000 max=132383 max_rank=3 min=127305 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=83222 max_load_reduction_pct=0.385993 moved_to_replicas=365524 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] avg=131072.0000 max=132383 max_rank=3 min=127305 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=83222 max_load_reduction_pct=0.385993 moved_to_replicas=365524 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] avg=131072.0000 max=132384 max_rank=2 min=126616 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86083 max_load_reduction_pct=0.394032 moved_to_replicas=311922 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] avg=131072.0000 max=132383 max_rank=3 min=127305 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=83222 max_load_reduction_pct=0.385993 moved_to_replicas=365524 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[131472, 132382, 132384, 132383, 130601, 132383, 125934, 131037] avg=131072.0000 max=132384 max_rank=2 min=125934 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=89621 max_load_reduction_pct=0.403689 moved_to_replicas=383071 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[67204, 130984, 218467, 140179, 66403, 163829, 98249, 163261] avg=131072.0000 max=218467 max_rank=2 min=66403 min_rank=4 imbalance_ratio=1.666771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[31627, 98284, 112871, 196596, 215605, 98445, 130799, 164349] avg=131072.0000 max=215605 max_rank=4 min=31627 min_rank=0 imbalance_ratio=1.644936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65792, 115182, 129911, 203069, 142193, 229285, 97556, 65588] avg=131072.0000 max=229285 max_rank=5 min=65588 min_rank=7 imbalance_ratio=1.749306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[67204, 130984, 218467, 140179, 66403, 163829, 98249, 163261] avg=131072.0000 max=218467 max_rank=2 min=66403 min_rank=4 imbalance_ratio=1.666771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65792, 115182, 129911, 203069, 142193, 229285, 97556, 65588] avg=131072.0000 max=229285 max_rank=5 min=65588 min_rank=7 imbalance_ratio=1.749306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[67204, 130984, 218467, 140179, 66403, 163829, 98249, 163261] avg=131072.0000 max=218467 max_rank=2 min=66403 min_rank=4 imbalance_ratio=1.666771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] avg=131072.0000 max=132384 max_rank=2 min=126616 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86083 max_load_reduction_pct=0.394032 moved_to_replicas=311922 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[67204, 130984, 218467, 140179, 66403, 163829, 98249, 163261] avg=131072.0000 max=218467 max_rank=2 min=66403 min_rank=4 imbalance_ratio=1.666771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] avg=131072.0000 max=132383 max_rank=3 min=127305 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=83222 max_load_reduction_pct=0.385993 moved_to_replicas=365524 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[31627, 98284, 112871, 196596, 215605, 98445, 130799, 164349] avg=131072.0000 max=215605 max_rank=4 min=31627 min_rank=0 imbalance_ratio=1.644936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] avg=131072.0000 max=132383 max_rank=3 min=128950 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96902 max_load_reduction_pct=0.422627 moved_to_replicas=347955 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] avg=131072.0000 max=132383 max_rank=3 min=128950 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96902 max_load_reduction_pct=0.422627 moved_to_replicas=347955 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] avg=131072.0000 max=132384 max_rank=2 min=126616 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86083 max_load_reduction_pct=0.394032 moved_to_replicas=311922 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] avg=131072.0000 max=132384 max_rank=2 min=126616 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86083 max_load_reduction_pct=0.394032 moved_to_replicas=311922 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] avg=131072.0000 max=132384 max_rank=2 min=126616 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86083 max_load_reduction_pct=0.394032 moved_to_replicas=311922 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[127305, 131042, 132382, 132383, 132382, 129900, 130799, 132383] avg=131072.0000 max=132383 max_rank=3 min=127305 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=83222 max_load_reduction_pct=0.385993 moved_to_replicas=365524 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65792, 115182, 129911, 203069, 142193, 229285, 97556, 65588] avg=131072.0000 max=229285 max_rank=5 min=65588 min_rank=7 imbalance_ratio=1.749306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[67204, 130984, 218467, 140179, 66403, 163829, 98249, 163261] avg=131072.0000 max=218467 max_rank=2 min=66403 min_rank=4 imbalance_ratio=1.666771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[98035, 34077, 166154, 131069, 163756, 197145, 65688, 192652] avg=131072.0000 max=197145 max_rank=5 min=34077 min_rank=1 imbalance_ratio=1.504097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[98035, 34077, 166154, 131069, 163756, 197145, 65688, 192652] avg=131072.0000 max=197145 max_rank=5 min=34077 min_rank=1 imbalance_ratio=1.504097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65792, 115182, 129911, 203069, 142193, 229285, 97556, 65588] avg=131072.0000 max=229285 max_rank=5 min=65588 min_rank=7 imbalance_ratio=1.749306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65792, 115182, 129911, 203069, 142193, 229285, 97556, 65588] avg=131072.0000 max=229285 max_rank=5 min=65588 min_rank=7 imbalance_ratio=1.749306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] avg=131072.0000 max=132383 max_rank=3 min=128950 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96902 max_load_reduction_pct=0.422627 moved_to_replicas=347955 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] avg=131072.0000 max=132384 max_rank=2 min=126616 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86083 max_load_reduction_pct=0.394032 moved_to_replicas=311922 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] avg=131072.0000 max=132383 max_rank=2 min=125209 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=64762 max_load_reduction_pct=0.328499 moved_to_replicas=379638 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[67204, 130984, 218467, 140179, 66403, 163829, 98249, 163261] avg=131072.0000 max=218467 max_rank=2 min=66403 min_rank=4 imbalance_ratio=1.666771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65792, 115182, 129911, 203069, 142193, 229285, 97556, 65588] avg=131072.0000 max=229285 max_rank=5 min=65588 min_rank=7 imbalance_ratio=1.749306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] avg=131072.0000 max=132383 max_rank=2 min=125209 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=64762 max_load_reduction_pct=0.328499 moved_to_replicas=379638 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] avg=131072.0000 max=132383 max_rank=3 min=128950 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96902 max_load_reduction_pct=0.422627 moved_to_replicas=347955 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] avg=131072.0000 max=132383 max_rank=3 min=128950 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96902 max_load_reduction_pct=0.422627 moved_to_replicas=347955 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[98035, 34077, 166154, 131069, 163756, 197145, 65688, 192652] avg=131072.0000 max=197145 max_rank=5 min=34077 min_rank=1 imbalance_ratio=1.504097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[130838, 130984, 132384, 132382, 130606, 132383, 126616, 132383] avg=131072.0000 max=132384 max_rank=2 min=126616 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=86083 max_load_reduction_pct=0.394032 moved_to_replicas=311922 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] avg=131072.0000 max=132383 max_rank=3 min=128950 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96902 max_load_reduction_pct=0.422627 moved_to_replicas=347955 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[45391, 152021, 98277, 130873, 131061, 32858, 228792, 229303] avg=131072.0000 max=229303 max_rank=7 min=32858 min_rank=5 imbalance_ratio=1.749443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65792, 115182, 129911, 203069, 142193, 229285, 97556, 65588] avg=131072.0000 max=229285 max_rank=5 min=65588 min_rank=7 imbalance_ratio=1.749306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[45391, 152021, 98277, 130873, 131061, 32858, 228792, 229303] avg=131072.0000 max=229303 max_rank=7 min=32858 min_rank=5 imbalance_ratio=1.749443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[98035, 34077, 166154, 131069, 163756, 197145, 65688, 192652] avg=131072.0000 max=197145 max_rank=5 min=34077 min_rank=1 imbalance_ratio=1.504097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[98035, 34077, 166154, 131069, 163756, 197145, 65688, 192652] avg=131072.0000 max=197145 max_rank=5 min=34077 min_rank=1 imbalance_ratio=1.504097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] avg=131072.0000 max=132383 max_rank=2 min=125209 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=64762 max_load_reduction_pct=0.328499 moved_to_replicas=379638 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] avg=131072.0000 max=132385 max_rank=1 min=129178 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96918 max_load_reduction_pct=0.422663 moved_to_replicas=424655 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] avg=131072.0000 max=132383 max_rank=3 min=128950 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96902 max_load_reduction_pct=0.422627 moved_to_replicas=347955 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65792, 115182, 129911, 203069, 142193, 229285, 97556, 65588] avg=131072.0000 max=229285 max_rank=5 min=65588 min_rank=7 imbalance_ratio=1.749306 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[98035, 34077, 166154, 131069, 163756, 197145, 65688, 192652] avg=131072.0000 max=197145 max_rank=5 min=34077 min_rank=1 imbalance_ratio=1.504097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] avg=131072.0000 max=132385 max_rank=1 min=129178 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96918 max_load_reduction_pct=0.422663 moved_to_replicas=424655 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] avg=131072.0000 max=132383 max_rank=2 min=125209 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=64762 max_load_reduction_pct=0.328499 moved_to_replicas=379638 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] avg=131072.0000 max=132383 max_rank=2 min=125209 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=64762 max_load_reduction_pct=0.328499 moved_to_replicas=379638 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[45391, 152021, 98277, 130873, 131061, 32858, 228792, 229303] avg=131072.0000 max=229303 max_rank=7 min=32858 min_rank=5 imbalance_ratio=1.749443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[131299, 130169, 129911, 132383, 132382, 132383, 128950, 131099] avg=131072.0000 max=132383 max_rank=3 min=128950 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=96902 max_load_reduction_pct=0.422627 moved_to_replicas=347955 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] avg=131072.0000 max=132383 max_rank=2 min=125209 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=64762 max_load_reduction_pct=0.328499 moved_to_replicas=379638 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[141931, 89322, 98362, 131056, 98177, 161939, 131217, 196572] avg=131072.0000 max=196572 max_rank=7 min=89322 min_rank=1 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[98035, 34077, 166154, 131069, 163756, 197145, 65688, 192652] avg=131072.0000 max=197145 max_rank=5 min=34077 min_rank=1 imbalance_ratio=1.504097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[141931, 89322, 98362, 131056, 98177, 161939, 131217, 196572] avg=131072.0000 max=196572 max_rank=7 min=89322 min_rank=1 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[45391, 152021, 98277, 130873, 131061, 32858, 228792, 229303] avg=131072.0000 max=229303 max_rank=7 min=32858 min_rank=5 imbalance_ratio=1.749443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] avg=131072.0000 max=132385 max_rank=1 min=129178 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96918 max_load_reduction_pct=0.422663 moved_to_replicas=424655 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[45391, 152021, 98277, 130873, 131061, 32858, 228792, 229303] avg=131072.0000 max=229303 max_rank=7 min=32858 min_rank=5 imbalance_ratio=1.749443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] avg=131072.0000 max=132384 max_rank=0 min=127918 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64188 max_load_reduction_pct=0.326537 moved_to_replicas=206962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] avg=131072.0000 max=132383 max_rank=2 min=125209 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=64762 max_load_reduction_pct=0.328499 moved_to_replicas=379638 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[98035, 34077, 166154, 131069, 163756, 197145, 65688, 192652] avg=131072.0000 max=197145 max_rank=5 min=34077 min_rank=1 imbalance_ratio=1.504097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] avg=131072.0000 max=132384 max_rank=0 min=127918 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64188 max_load_reduction_pct=0.326537 moved_to_replicas=206962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[45391, 152021, 98277, 130873, 131061, 32858, 228792, 229303] avg=131072.0000 max=229303 max_rank=7 min=32858 min_rank=5 imbalance_ratio=1.749443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] avg=131072.0000 max=132385 max_rank=1 min=129178 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96918 max_load_reduction_pct=0.422663 moved_to_replicas=424655 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] avg=131072.0000 max=132385 max_rank=1 min=129178 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96918 max_load_reduction_pct=0.422663 moved_to_replicas=424655 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[141931, 89322, 98362, 131056, 98177, 161939, 131217, 196572] avg=131072.0000 max=196572 max_rank=7 min=89322 min_rank=1 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[130789, 131979, 132383, 131069, 132383, 132383, 125209, 132381] avg=131072.0000 max=132383 max_rank=2 min=125209 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=64762 max_load_reduction_pct=0.328499 moved_to_replicas=379638 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[98538, 212925, 60285, 164355, 98375, 217608, 65579, 130911] avg=131072.0000 max=217608 max_rank=5 min=60285 min_rank=2 imbalance_ratio=1.660217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] avg=131072.0000 max=132385 max_rank=1 min=129178 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96918 max_load_reduction_pct=0.422663 moved_to_replicas=424655 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[45391, 152021, 98277, 130873, 131061, 32858, 228792, 229303] avg=131072.0000 max=229303 max_rank=7 min=32858 min_rank=5 imbalance_ratio=1.749443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[98538, 212925, 60285, 164355, 98375, 217608, 65579, 130911] avg=131072.0000 max=217608 max_rank=5 min=60285 min_rank=2 imbalance_ratio=1.660217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] avg=131072.0000 max=132384 max_rank=0 min=127918 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64188 max_load_reduction_pct=0.326537 moved_to_replicas=206962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[141931, 89322, 98362, 131056, 98177, 161939, 131217, 196572] avg=131072.0000 max=196572 max_rank=7 min=89322 min_rank=1 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[141931, 89322, 98362, 131056, 98177, 161939, 131217, 196572] avg=131072.0000 max=196572 max_rank=7 min=89322 min_rank=1 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] avg=131072.0000 max=132386 max_rank=2 min=125917 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=85222 max_load_reduction_pct=0.391631 moved_to_replicas=393898 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] avg=131072.0000 max=132385 max_rank=1 min=129178 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96918 max_load_reduction_pct=0.422663 moved_to_replicas=424655 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] avg=131072.0000 max=132386 max_rank=2 min=125917 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=85222 max_load_reduction_pct=0.391631 moved_to_replicas=393898 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[45391, 152021, 98277, 130873, 131061, 32858, 228792, 229303] avg=131072.0000 max=229303 max_rank=7 min=32858 min_rank=5 imbalance_ratio=1.749443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] avg=131072.0000 max=132384 max_rank=0 min=127918 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64188 max_load_reduction_pct=0.326537 moved_to_replicas=206962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[141931, 89322, 98362, 131056, 98177, 161939, 131217, 196572] avg=131072.0000 max=196572 max_rank=7 min=89322 min_rank=1 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] avg=131072.0000 max=132384 max_rank=0 min=127918 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64188 max_load_reduction_pct=0.326537 moved_to_replicas=206962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[98538, 212925, 60285, 164355, 98375, 217608, 65579, 130911] avg=131072.0000 max=217608 max_rank=5 min=60285 min_rank=2 imbalance_ratio=1.660217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[99870, 133601, 55583, 130933, 155410, 240725, 166868, 65586] avg=131072.0000 max=240725 max_rank=5 min=55583 min_rank=2 imbalance_ratio=1.836586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[130538, 132385, 129178, 130873, 131061, 129775, 132383, 132383] avg=131072.0000 max=132385 max_rank=1 min=129178 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96918 max_load_reduction_pct=0.422663 moved_to_replicas=424655 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[141931, 89322, 98362, 131056, 98177, 161939, 131217, 196572] avg=131072.0000 max=196572 max_rank=7 min=89322 min_rank=1 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] avg=131072.0000 max=132384 max_rank=0 min=127918 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64188 max_load_reduction_pct=0.326537 moved_to_replicas=206962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[99870, 133601, 55583, 130933, 155410, 240725, 166868, 65586] avg=131072.0000 max=240725 max_rank=5 min=55583 min_rank=2 imbalance_ratio=1.836586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] avg=131072.0000 max=132386 max_rank=2 min=125917 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=85222 max_load_reduction_pct=0.391631 moved_to_replicas=393898 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[98538, 212925, 60285, 164355, 98375, 217608, 65579, 130911] avg=131072.0000 max=217608 max_rank=5 min=60285 min_rank=2 imbalance_ratio=1.660217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] avg=131072.0000 max=132385 max_rank=5 min=123345 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108340 max_load_reduction_pct=0.450057 moved_to_replicas=336189 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[98538, 212925, 60285, 164355, 98375, 217608, 65579, 130911] avg=131072.0000 max=217608 max_rank=5 min=60285 min_rank=2 imbalance_ratio=1.660217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] avg=131072.0000 max=132384 max_rank=0 min=127918 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64188 max_load_reduction_pct=0.326537 moved_to_replicas=206962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] avg=131072.0000 max=132385 max_rank=5 min=123345 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108340 max_load_reduction_pct=0.450057 moved_to_replicas=336189 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[141931, 89322, 98362, 131056, 98177, 161939, 131217, 196572] avg=131072.0000 max=196572 max_rank=7 min=89322 min_rank=1 imbalance_ratio=1.499725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] avg=131072.0000 max=132386 max_rank=2 min=125917 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=85222 max_load_reduction_pct=0.391631 moved_to_replicas=393898 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[99870, 133601, 55583, 130933, 155410, 240725, 166868, 65586] avg=131072.0000 max=240725 max_rank=5 min=55583 min_rank=2 imbalance_ratio=1.836586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[98538, 212925, 60285, 164355, 98375, 217608, 65579, 130911] avg=131072.0000 max=217608 max_rank=5 min=60285 min_rank=2 imbalance_ratio=1.660217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] avg=131072.0000 max=132386 max_rank=2 min=125917 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=85222 max_load_reduction_pct=0.391631 moved_to_replicas=393898 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[226398, 165045, 66113, 98429, 132520, 98650, 130201, 131220] avg=131072.0000 max=226398 max_rank=0 min=66113 min_rank=2 imbalance_ratio=1.727280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[132384, 131624, 127918, 131056, 129611, 132383, 131217, 132383] avg=131072.0000 max=132384 max_rank=0 min=127918 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64188 max_load_reduction_pct=0.326537 moved_to_replicas=206962 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] avg=131072.0000 max=132385 max_rank=5 min=123345 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108340 max_load_reduction_pct=0.450057 moved_to_replicas=336189 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[226398, 165045, 66113, 98429, 132520, 98650, 130201, 131220] avg=131072.0000 max=226398 max_rank=0 min=66113 min_rank=2 imbalance_ratio=1.727280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[98538, 212925, 60285, 164355, 98375, 217608, 65579, 130911] avg=131072.0000 max=217608 max_rank=5 min=60285 min_rank=2 imbalance_ratio=1.660217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] avg=131072.0000 max=132386 max_rank=2 min=125917 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=85222 max_load_reduction_pct=0.391631 moved_to_replicas=393898 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[99870, 133601, 55583, 130933, 155410, 240725, 166868, 65586] avg=131072.0000 max=240725 max_rank=5 min=55583 min_rank=2 imbalance_ratio=1.836586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] avg=131072.0000 max=132383 max_rank=1 min=127298 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=94015 max_load_reduction_pct=0.415264 moved_to_replicas=254653 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[99870, 133601, 55583, 130933, 155410, 240725, 166868, 65586] avg=131072.0000 max=240725 max_rank=5 min=55583 min_rank=2 imbalance_ratio=1.836586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] avg=131072.0000 max=132383 max_rank=1 min=127298 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=94015 max_load_reduction_pct=0.415264 moved_to_replicas=254653 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] avg=131072.0000 max=132386 max_rank=2 min=125917 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=85222 max_load_reduction_pct=0.391631 moved_to_replicas=393898 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[98538, 212925, 60285, 164355, 98375, 217608, 65579, 130911] avg=131072.0000 max=217608 max_rank=5 min=60285 min_rank=2 imbalance_ratio=1.660217 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] avg=131072.0000 max=132385 max_rank=5 min=123345 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108340 max_load_reduction_pct=0.450057 moved_to_replicas=336189 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[226398, 165045, 66113, 98429, 132520, 98650, 130201, 131220] avg=131072.0000 max=226398 max_rank=0 min=66113 min_rank=2 imbalance_ratio=1.727280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] avg=131072.0000 max=132385 max_rank=5 min=123345 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108340 max_load_reduction_pct=0.450057 moved_to_replicas=336189 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[99870, 133601, 55583, 130933, 155410, 240725, 166868, 65586] avg=131072.0000 max=240725 max_rank=5 min=55583 min_rank=2 imbalance_ratio=1.836586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[66203, 107914, 130798, 163814, 125603, 229263, 126584, 98397] avg=131072.0000 max=229263 max_rank=5 min=66203 min_rank=0 imbalance_ratio=1.749138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[125917, 132382, 132386, 132383, 131130, 132382, 131085, 130911] avg=131072.0000 max=132386 max_rank=2 min=125917 min_rank=0 imbalance_ratio=1.010025 max_load_reduction=85222 max_load_reduction_pct=0.391631 moved_to_replicas=393898 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] avg=131072.0000 max=132383 max_rank=1 min=127298 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=94015 max_load_reduction_pct=0.415264 moved_to_replicas=254653 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[66203, 107914, 130798, 163814, 125603, 229263, 126584, 98397] avg=131072.0000 max=229263 max_rank=5 min=66203 min_rank=0 imbalance_ratio=1.749138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[99870, 133601, 55583, 130933, 155410, 240725, 166868, 65586] avg=131072.0000 max=240725 max_rank=5 min=55583 min_rank=2 imbalance_ratio=1.836586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[226398, 165045, 66113, 98429, 132520, 98650, 130201, 131220] avg=131072.0000 max=226398 max_rank=0 min=66113 min_rank=2 imbalance_ratio=1.727280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] avg=131072.0000 max=132384 max_rank=4 min=126766 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96879 max_load_reduction_pct=0.422567 moved_to_replicas=256740 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] avg=131072.0000 max=132385 max_rank=5 min=123345 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108340 max_load_reduction_pct=0.450057 moved_to_replicas=336189 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[226398, 165045, 66113, 98429, 132520, 98650, 130201, 131220] avg=131072.0000 max=226398 max_rank=0 min=66113 min_rank=2 imbalance_ratio=1.727280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] avg=131072.0000 max=132384 max_rank=4 min=126766 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96879 max_load_reduction_pct=0.422567 moved_to_replicas=256740 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] avg=131072.0000 max=132385 max_rank=5 min=123345 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108340 max_load_reduction_pct=0.450057 moved_to_replicas=336189 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[99870, 133601, 55583, 130933, 155410, 240725, 166868, 65586] avg=131072.0000 max=240725 max_rank=5 min=55583 min_rank=2 imbalance_ratio=1.836586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[66203, 107914, 130798, 163814, 125603, 229263, 126584, 98397] avg=131072.0000 max=229263 max_rank=5 min=66203 min_rank=0 imbalance_ratio=1.749138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] avg=131072.0000 max=132383 max_rank=1 min=127298 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=94015 max_load_reduction_pct=0.415264 moved_to_replicas=254653 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] avg=131072.0000 max=132383 max_rank=1 min=127298 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=94015 max_load_reduction_pct=0.415264 moved_to_replicas=254653 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[131442, 196926, 33049, 65094, 163729, 168517, 196999, 92820] avg=131072.0000 max=196999 max_rank=6 min=33049 min_rank=2 imbalance_ratio=1.502983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[226398, 165045, 66113, 98429, 132520, 98650, 130201, 131220] avg=131072.0000 max=226398 max_rank=0 min=66113 min_rank=2 imbalance_ratio=1.727280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[131442, 196926, 33049, 65094, 163729, 168517, 196999, 92820] avg=131072.0000 max=196999 max_rank=6 min=33049 min_rank=2 imbalance_ratio=1.502983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[132383, 132383, 123345, 130933, 132383, 132385, 132383, 132381] avg=131072.0000 max=132385 max_rank=5 min=123345 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=108340 max_load_reduction_pct=0.450057 moved_to_replicas=336189 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] avg=131072.0000 max=132384 max_rank=4 min=126766 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96879 max_load_reduction_pct=0.422567 moved_to_replicas=256740 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] avg=131072.0000 max=132383 max_rank=1 min=125534 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64616 max_load_reduction_pct=0.328002 moved_to_replicas=393266 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[226398, 165045, 66113, 98429, 132520, 98650, 130201, 131220] avg=131072.0000 max=226398 max_rank=0 min=66113 min_rank=2 imbalance_ratio=1.727280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[66203, 107914, 130798, 163814, 125603, 229263, 126584, 98397] avg=131072.0000 max=229263 max_rank=5 min=66203 min_rank=0 imbalance_ratio=1.749138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] avg=131072.0000 max=132383 max_rank=1 min=127298 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=94015 max_load_reduction_pct=0.415264 moved_to_replicas=254653 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] avg=131072.0000 max=132383 max_rank=1 min=125534 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64616 max_load_reduction_pct=0.328002 moved_to_replicas=393266 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[66203, 107914, 130798, 163814, 125603, 229263, 126584, 98397] avg=131072.0000 max=229263 max_rank=5 min=66203 min_rank=0 imbalance_ratio=1.749138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[131442, 196926, 33049, 65094, 163729, 168517, 196999, 92820] avg=131072.0000 max=196999 max_rank=6 min=33049 min_rank=2 imbalance_ratio=1.502983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] avg=131072.0000 max=132383 max_rank=1 min=127298 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=94015 max_load_reduction_pct=0.415264 moved_to_replicas=254653 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[226398, 165045, 66113, 98429, 132520, 98650, 130201, 131220] avg=131072.0000 max=226398 max_rank=0 min=66113 min_rank=2 imbalance_ratio=1.727280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] avg=131072.0000 max=132384 max_rank=4 min=126766 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96879 max_load_reduction_pct=0.422567 moved_to_replicas=256740 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[98139, 164182, 70631, 130961, 126011, 163754, 130678, 164220] avg=131072.0000 max=164220 max_rank=7 min=70631 min_rank=2 imbalance_ratio=1.252899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] avg=131072.0000 max=132384 max_rank=4 min=126766 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96879 max_load_reduction_pct=0.422567 moved_to_replicas=256740 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[98139, 164182, 70631, 130961, 126011, 163754, 130678, 164220] avg=131072.0000 max=164220 max_rank=7 min=70631 min_rank=2 imbalance_ratio=1.252899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[66203, 107914, 130798, 163814, 125603, 229263, 126584, 98397] avg=131072.0000 max=229263 max_rank=5 min=66203 min_rank=0 imbalance_ratio=1.749138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] avg=131072.0000 max=132383 max_rank=1 min=125534 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64616 max_load_reduction_pct=0.328002 moved_to_replicas=393266 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[132381, 132383, 131529, 131181, 132383, 127298, 130201, 131220] avg=131072.0000 max=132383 max_rank=1 min=127298 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=94015 max_load_reduction_pct=0.415264 moved_to_replicas=254653 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] avg=131072.0000 max=132383 max_rank=1 min=127467 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31837 max_load_reduction_pct=0.193868 moved_to_replicas=189502 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[66203, 107914, 130798, 163814, 125603, 229263, 126584, 98397] avg=131072.0000 max=229263 max_rank=5 min=66203 min_rank=0 imbalance_ratio=1.749138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[131442, 196926, 33049, 65094, 163729, 168517, 196999, 92820] avg=131072.0000 max=196999 max_rank=6 min=33049 min_rank=2 imbalance_ratio=1.502983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] avg=131072.0000 max=132383 max_rank=1 min=127467 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31837 max_load_reduction_pct=0.193868 moved_to_replicas=189502 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] avg=131072.0000 max=132384 max_rank=4 min=126766 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96879 max_load_reduction_pct=0.422567 moved_to_replicas=256740 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[131442, 196926, 33049, 65094, 163729, 168517, 196999, 92820] avg=131072.0000 max=196999 max_rank=6 min=33049 min_rank=2 imbalance_ratio=1.502983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[98139, 164182, 70631, 130961, 126011, 163754, 130678, 164220] avg=131072.0000 max=164220 max_rank=7 min=70631 min_rank=2 imbalance_ratio=1.252899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] avg=131072.0000 max=132384 max_rank=4 min=126766 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96879 max_load_reduction_pct=0.422567 moved_to_replicas=256740 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[66203, 107914, 130798, 163814, 125603, 229263, 126584, 98397] avg=131072.0000 max=229263 max_rank=5 min=66203 min_rank=0 imbalance_ratio=1.749138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] avg=131072.0000 max=132383 max_rank=1 min=125534 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64616 max_load_reduction_pct=0.328002 moved_to_replicas=393266 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] avg=131072.0000 max=132383 max_rank=1 min=125534 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64616 max_load_reduction_pct=0.328002 moved_to_replicas=393266 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[131442, 196926, 33049, 65094, 163729, 168517, 196999, 92820] avg=131072.0000 max=196999 max_rank=6 min=33049 min_rank=2 imbalance_ratio=1.502983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] avg=131072.0000 max=132383 max_rank=1 min=127467 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31837 max_load_reduction_pct=0.193868 moved_to_replicas=189502 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[130330, 132382, 130798, 132383, 132384, 132383, 126766, 131150] avg=131072.0000 max=132384 max_rank=4 min=126766 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=96879 max_load_reduction_pct=0.422567 moved_to_replicas=256740 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[131442, 196926, 33049, 65094, 163729, 168517, 196999, 92820] avg=131072.0000 max=196999 max_rank=6 min=33049 min_rank=2 imbalance_ratio=1.502983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[98139, 164182, 70631, 130961, 126011, 163754, 130678, 164220] avg=131072.0000 max=164220 max_rank=7 min=70631 min_rank=2 imbalance_ratio=1.252899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] avg=131072.0000 max=132383 max_rank=1 min=125534 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64616 max_load_reduction_pct=0.328002 moved_to_replicas=393266 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[98139, 164182, 70631, 130961, 126011, 163754, 130678, 164220] avg=131072.0000 max=164220 max_rank=7 min=70631 min_rank=2 imbalance_ratio=1.252899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] avg=131072.0000 max=132383 max_rank=1 min=125534 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64616 max_load_reduction_pct=0.328002 moved_to_replicas=393266 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[131442, 196926, 33049, 65094, 163729, 168517, 196999, 92820] avg=131072.0000 max=196999 max_rank=6 min=33049 min_rank=2 imbalance_ratio=1.502983 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] avg=131072.0000 max=132383 max_rank=1 min=127467 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31837 max_load_reduction_pct=0.193868 moved_to_replicas=189502 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] avg=131072.0000 max=132383 max_rank=1 min=127467 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31837 max_load_reduction_pct=0.193868 moved_to_replicas=189502 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[98139, 164182, 70631, 130961, 126011, 163754, 130678, 164220] avg=131072.0000 max=164220 max_rank=7 min=70631 min_rank=2 imbalance_ratio=1.252899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[131442, 132383, 132383, 129685, 132383, 132383, 132383, 125534] avg=131072.0000 max=132383 max_rank=1 min=125534 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64616 max_load_reduction_pct=0.328002 moved_to_replicas=393266 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[98139, 164182, 70631, 130961, 126011, 163754, 130678, 164220] avg=131072.0000 max=164220 max_rank=7 min=70631 min_rank=2 imbalance_ratio=1.252899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] avg=131072.0000 max=132383 max_rank=1 min=127467 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31837 max_load_reduction_pct=0.193868 moved_to_replicas=189502 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] avg=131072.0000 max=132383 max_rank=1 min=127467 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31837 max_load_reduction_pct=0.193868 moved_to_replicas=189502 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[98139, 164182, 70631, 130961, 126011, 163754, 130678, 164220] avg=131072.0000 max=164220 max_rank=7 min=70631 min_rank=2 imbalance_ratio=1.252899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[129938, 132383, 132383, 130961, 127467, 132383, 130678, 132383] avg=131072.0000 max=132383 max_rank=1 min=127467 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=31837 max_load_reduction_pct=0.193868 moved_to_replicas=189502 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=4209196 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=999040 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=4206956 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1002496 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=4179438 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1025536 peak_demand=132386 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=4174425 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1032064 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=4222891 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=984704 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=4159596 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1046912 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=4195368 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1011200 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=4206562 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1001216 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:17:55] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 578.4 | throughput per GPU (TFLOP/s/GPU): 576.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.323610E+00 | loss scale: 1.0 | grad norm: 5.520 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=163840 expected_recv=257726 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=163840 expected_recv=257286 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=163840 expected_recv=258310 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=163840 expected_recv=257726 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=163840 expected_recv=257286 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=163840 expected_recv=258310 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=259712 slack_slots=67968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=260352 slack_slots=67328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=260608 slack_slots=67072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264301 expected_overflow=0 exact_recv_per_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=163840 expected_recv=247638 expected_overflow=0 exact_recv_per_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264301 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=163840 expected_recv=247638 actual_valid_recv=249856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=249856 slack_slots=77824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=163840 expected_recv=261384 expected_overflow=0 exact_recv_per_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=163840 expected_recv=261384 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=163840 expected_recv=254475 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=163840 expected_recv=254475 actual_valid_recv=257152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261168 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=257152 slack_slots=70528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261132 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261168 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261132 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261310 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261310 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=163840 expected_recv=262205 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=163840 expected_recv=259769 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=163840 expected_recv=262205 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=163840 expected_recv=259769 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=163840 expected_recv=257124 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=163840 expected_recv=258991 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=163840 expected_recv=264346 expected_overflow=0 exact_recv_per_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=163840 expected_recv=257124 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262536 expected_overflow=0 exact_recv_per_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=163840 expected_recv=258991 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=163840 expected_recv=264346 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=163840 expected_recv=261923 expected_overflow=0 exact_recv_per_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262536 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=163840 expected_recv=261923 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=163840 expected_recv=260144 expected_overflow=0 exact_recv_per_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=163840 expected_recv=256553 expected_overflow=0 exact_recv_per_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=163840 expected_recv=260144 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=163840 expected_recv=256553 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=259200 slack_slots=68480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262120 expected_overflow=0 exact_recv_per_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262120 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=163840 expected_recv=257055 expected_overflow=0 exact_recv_per_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=163840 expected_recv=257055 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=259584 slack_slots=68096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=163840 expected_recv=257150 expected_overflow=0 exact_recv_per_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=163840 expected_recv=257150 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=163840 expected_recv=261915 expected_overflow=0 exact_recv_per_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=259712 slack_slots=67968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=163840 expected_recv=256936 expected_overflow=0 exact_recv_per_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=163840 expected_recv=261966 expected_overflow=0 exact_recv_per_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=163840 expected_recv=261334 expected_overflow=0 exact_recv_per_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=163840 expected_recv=261915 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=163840 expected_recv=256936 actual_valid_recv=258944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262881 expected_overflow=0 exact_recv_per_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=163840 expected_recv=261966 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=163840 expected_recv=261334 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=258944 slack_slots=68736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262881 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262019 expected_overflow=0 exact_recv_per_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262019 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262332 expected_overflow=0 exact_recv_per_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262116 expected_overflow=0 exact_recv_per_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262782 expected_overflow=0 exact_recv_per_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262183 expected_overflow=0 exact_recv_per_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262332 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262116 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262782 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262183 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=163840 expected_recv=264772 expected_overflow=0 exact_recv_per_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=163840 expected_recv=264772 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=163840 expected_recv=260932 expected_overflow=0 exact_recv_per_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=163840 expected_recv=250775 expected_overflow=0 exact_recv_per_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=163840 expected_recv=261092 expected_overflow=0 exact_recv_per_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=163840 expected_recv=260932 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=163840 expected_recv=261903 expected_overflow=0 exact_recv_per_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262342 expected_overflow=0 exact_recv_per_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=163840 expected_recv=250775 actual_valid_recv=253184 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=163840 expected_recv=261092 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=163840 expected_recv=259384 expected_overflow=0 exact_recv_per_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=163840 expected_recv=261903 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=253184 slack_slots=74496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262342 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=163840 expected_recv=259384 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=163840 expected_recv=259738 expected_overflow=0 exact_recv_per_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=163840 expected_recv=258920 expected_overflow=0 exact_recv_per_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=163840 expected_recv=259543 expected_overflow=0 exact_recv_per_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=163840 expected_recv=259738 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=163840 expected_recv=258920 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=163840 expected_recv=259543 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=260864 slack_slots=66816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=163840 expected_recv=252491 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=163840 expected_recv=262150 expected_overflow=0 exact_recv_per_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=163840 expected_recv=257280 expected_overflow=0 exact_recv_per_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=163840 expected_recv=252491 actual_valid_recv=254976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=163840 expected_recv=262150 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=163840 expected_recv=257280 actual_valid_recv=259328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=254976 slack_slots=72704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=259328 slack_slots=68352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=163840 expected_recv=256976 expected_overflow=0 exact_recv_per_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=163840 expected_recv=260769 expected_overflow=0 exact_recv_per_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=163840 expected_recv=256976 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=163840 expected_recv=262023 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=163840 expected_recv=260769 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=163840 expected_recv=258081 expected_overflow=0 exact_recv_per_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=163840 expected_recv=258809 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=259456 slack_slots=68224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=163840 expected_recv=262023 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=163840 expected_recv=258081 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=163840 expected_recv=258809 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=260736 slack_slots=66944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=163840 expected_recv=259244 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=163840 expected_recv=259244 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=163840 expected_recv=260443 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=163840 expected_recv=262262 expected_overflow=0 exact_recv_per_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=163840 expected_recv=260443 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262269 expected_overflow=0 exact_recv_per_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=163840 expected_recv=261576 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=163840 expected_recv=262262 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262269 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=163840 expected_recv=261576 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=163840 expected_recv=255593 expected_overflow=0 exact_recv_per_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=163840 expected_recv=255593 actual_valid_recv=257280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=257280 slack_slots=70400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262197 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262419 expected_overflow=0 exact_recv_per_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=163840 expected_recv=259394 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=163840 expected_recv=259942 expected_overflow=0 exact_recv_per_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262419 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262197 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=163840 expected_recv=259394 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=163840 expected_recv=259942 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=163840 expected_recv=261650 expected_overflow=0 exact_recv_per_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262114 expected_overflow=0 exact_recv_per_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=163840 expected_recv=261650 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262114 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=163840 expected_recv=259742 expected_overflow=0 exact_recv_per_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=163840 expected_recv=259742 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=163840 expected_recv=258000 expected_overflow=0 exact_recv_per_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=261120 slack_slots=66560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=163840 expected_recv=260284 expected_overflow=0 exact_recv_per_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=163840 expected_recv=259423 expected_overflow=0 exact_recv_per_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=163840 expected_recv=260284 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=163840 expected_recv=258000 actual_valid_recv=260224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=163840 expected_recv=259423 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=163840 expected_recv=264229 expected_overflow=0 exact_recv_per_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=260224 slack_slots=67456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=163840 expected_recv=259615 expected_overflow=0 exact_recv_per_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=163840 expected_recv=264229 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=163840 expected_recv=259615 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=261760 slack_slots=65920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262272 expected_overflow=0 exact_recv_per_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[152783, 145996, 115569, 97024, 148127, 89488, 135215, 164374] avg=131072.0000 max=164374 max_rank=7 min=89488 min_rank=5 imbalance_ratio=1.254074 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262272 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[152783, 145996, 115569, 97024, 148127, 89488, 135215, 164374] avg=131072.0000 max=164374 max_rank=7 min=89488 min_rank=5 imbalance_ratio=1.254074 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[152783, 145996, 115569, 97024, 148127, 89488, 135215, 164374] avg=131072.0000 max=164374 max_rank=7 min=89488 min_rank=5 imbalance_ratio=1.254074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[152783, 145996, 115569, 97024, 148127, 89488, 135215, 164374] avg=131072.0000 max=164374 max_rank=7 min=89488 min_rank=5 imbalance_ratio=1.254074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=128241 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.194617 moved_to_replicas=165336 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=128241 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.194617 moved_to_replicas=165336 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[152783, 145996, 115569, 97024, 148127, 89488, 135215, 164374] avg=131072.0000 max=164374 max_rank=7 min=89488 min_rank=5 imbalance_ratio=1.254074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=128241 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.194617 moved_to_replicas=165336 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262184 expected_overflow=0 exact_recv_per_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=163840 expected_recv=258826 expected_overflow=0 exact_recv_per_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=128241 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.194617 moved_to_replicas=165336 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262184 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=163840 expected_recv=258826 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[152783, 145996, 115569, 97024, 148127, 89488, 135215, 164374] avg=131072.0000 max=164374 max_rank=7 min=89488 min_rank=5 imbalance_ratio=1.254074 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=261120 slack_slots=66560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[94418, 161194, 90207, 173822, 142001, 104628, 160599, 121707] avg=131072.0000 max=173822 max_rank=3 min=90207 min_rank=2 imbalance_ratio=1.326157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=128241 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.194617 moved_to_replicas=165336 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[94418, 161194, 90207, 173822, 142001, 104628, 160599, 121707] avg=131072.0000 max=173822 max_rank=3 min=90207 min_rank=2 imbalance_ratio=1.326157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[94418, 161194, 90207, 173822, 142001, 104628, 160599, 121707] avg=131072.0000 max=173822 max_rank=3 min=90207 min_rank=2 imbalance_ratio=1.326157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=128241 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.194617 moved_to_replicas=165336 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[94418, 161194, 90207, 173822, 142001, 104628, 160599, 121707] avg=131072.0000 max=173822 max_rank=3 min=90207 min_rank=2 imbalance_ratio=1.326157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] avg=131072.0000 max=132384 max_rank=5 min=122959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41438 max_load_reduction_pct=0.238393 moved_to_replicas=217449 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] avg=131072.0000 max=132384 max_rank=5 min=122959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41438 max_load_reduction_pct=0.238393 moved_to_replicas=217449 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] avg=131072.0000 max=132384 max_rank=5 min=122959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41438 max_load_reduction_pct=0.238393 moved_to_replicas=217449 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[152783, 145996, 115569, 97024, 148127, 89488, 135215, 164374] avg=131072.0000 max=164374 max_rank=7 min=89488 min_rank=5 imbalance_ratio=1.254074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[152783, 145996, 115569, 97024, 148127, 89488, 135215, 164374] avg=131072.0000 max=164374 max_rank=7 min=89488 min_rank=5 imbalance_ratio=1.254074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[94418, 161194, 90207, 173822, 142001, 104628, 160599, 121707] avg=131072.0000 max=173822 max_rank=3 min=90207 min_rank=2 imbalance_ratio=1.326157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] avg=131072.0000 max=132384 max_rank=5 min=122959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41438 max_load_reduction_pct=0.238393 moved_to_replicas=217449 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[146975, 188862, 102981, 221606, 130742, 139291, 43578, 74541] avg=131072.0000 max=221606 max_rank=3 min=43578 min_rank=6 imbalance_ratio=1.690720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[94418, 161194, 90207, 173822, 142001, 104628, 160599, 121707] avg=131072.0000 max=173822 max_rank=3 min=90207 min_rank=2 imbalance_ratio=1.326157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[146975, 188862, 102981, 221606, 130742, 139291, 43578, 74541] avg=131072.0000 max=221606 max_rank=3 min=43578 min_rank=6 imbalance_ratio=1.690720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=128241 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.194617 moved_to_replicas=165336 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[132383, 132382, 129183, 128241, 132382, 129238, 132383, 132384] avg=131072.0000 max=132384 max_rank=7 min=128241 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=31990 max_load_reduction_pct=0.194617 moved_to_replicas=165336 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] avg=131072.0000 max=132384 max_rank=5 min=122959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41438 max_load_reduction_pct=0.238393 moved_to_replicas=217449 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[146975, 188862, 102981, 221606, 130742, 139291, 43578, 74541] avg=131072.0000 max=221606 max_rank=3 min=43578 min_rank=6 imbalance_ratio=1.690720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] avg=131072.0000 max=132384 max_rank=1 min=127135 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=89222 max_load_reduction_pct=0.402615 moved_to_replicas=330644 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[146975, 188862, 102981, 221606, 130742, 139291, 43578, 74541] avg=131072.0000 max=221606 max_rank=3 min=43578 min_rank=6 imbalance_ratio=1.690720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] avg=131072.0000 max=132384 max_rank=5 min=122959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41438 max_load_reduction_pct=0.238393 moved_to_replicas=217449 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] avg=131072.0000 max=132384 max_rank=1 min=127135 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=89222 max_load_reduction_pct=0.402615 moved_to_replicas=330644 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] avg=131072.0000 max=132384 max_rank=1 min=127135 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=89222 max_load_reduction_pct=0.402615 moved_to_replicas=330644 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] avg=131072.0000 max=132384 max_rank=1 min=127135 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=89222 max_load_reduction_pct=0.402615 moved_to_replicas=330644 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[146975, 188862, 102981, 221606, 130742, 139291, 43578, 74541] avg=131072.0000 max=221606 max_rank=3 min=43578 min_rank=6 imbalance_ratio=1.690720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[94418, 161194, 90207, 173822, 142001, 104628, 160599, 121707] avg=131072.0000 max=173822 max_rank=3 min=90207 min_rank=2 imbalance_ratio=1.326157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[94418, 161194, 90207, 173822, 142001, 104628, 160599, 121707] avg=131072.0000 max=173822 max_rank=3 min=90207 min_rank=2 imbalance_ratio=1.326157 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[163708, 153570, 163721, 163876, 98254, 131090, 65634, 108723] avg=131072.0000 max=163876 max_rank=3 min=65634 min_rank=6 imbalance_ratio=1.250275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[163708, 153570, 163721, 163876, 98254, 131090, 65634, 108723] avg=131072.0000 max=163876 max_rank=3 min=65634 min_rank=6 imbalance_ratio=1.250275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[146975, 188862, 102981, 221606, 130742, 139291, 43578, 74541] avg=131072.0000 max=221606 max_rank=3 min=43578 min_rank=6 imbalance_ratio=1.690720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] avg=131072.0000 max=132384 max_rank=1 min=127135 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=89222 max_load_reduction_pct=0.402615 moved_to_replicas=330644 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[163708, 153570, 163721, 163876, 98254, 131090, 65634, 108723] avg=131072.0000 max=163876 max_rank=3 min=65634 min_rank=6 imbalance_ratio=1.250275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] avg=131072.0000 max=132384 max_rank=5 min=122959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41438 max_load_reduction_pct=0.238393 moved_to_replicas=217449 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[163708, 153570, 163721, 163876, 98254, 131090, 65634, 108723] avg=131072.0000 max=163876 max_rank=3 min=65634 min_rank=6 imbalance_ratio=1.250275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] avg=131072.0000 max=132383 max_rank=0 min=128465 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31493 max_load_reduction_pct=0.192176 moved_to_replicas=231665 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[131917, 132383, 122959, 132382, 132383, 132384, 132382, 131786] avg=131072.0000 max=132384 max_rank=5 min=122959 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=41438 max_load_reduction_pct=0.238393 moved_to_replicas=217449 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] avg=131072.0000 max=132383 max_rank=0 min=128465 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31493 max_load_reduction_pct=0.192176 moved_to_replicas=231665 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] avg=131072.0000 max=132384 max_rank=1 min=127135 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=89222 max_load_reduction_pct=0.402615 moved_to_replicas=330644 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] avg=131072.0000 max=132383 max_rank=0 min=128465 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31493 max_load_reduction_pct=0.192176 moved_to_replicas=231665 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] avg=131072.0000 max=132383 max_rank=0 min=128465 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31493 max_load_reduction_pct=0.192176 moved_to_replicas=231665 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[163708, 153570, 163721, 163876, 98254, 131090, 65634, 108723] avg=131072.0000 max=163876 max_rank=3 min=65634 min_rank=6 imbalance_ratio=1.250275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[100039, 98223, 131300, 129387, 131039, 103358, 153255, 201975] avg=131072.0000 max=201975 max_rank=7 min=98223 min_rank=1 imbalance_ratio=1.540947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[146975, 188862, 102981, 221606, 130742, 139291, 43578, 74541] avg=131072.0000 max=221606 max_rank=3 min=43578 min_rank=6 imbalance_ratio=1.690720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[100039, 98223, 131300, 129387, 131039, 103358, 153255, 201975] avg=131072.0000 max=201975 max_rank=7 min=98223 min_rank=1 imbalance_ratio=1.540947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[146975, 188862, 102981, 221606, 130742, 139291, 43578, 74541] avg=131072.0000 max=221606 max_rank=3 min=43578 min_rank=6 imbalance_ratio=1.690720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] avg=131072.0000 max=132383 max_rank=0 min=128465 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31493 max_load_reduction_pct=0.192176 moved_to_replicas=231665 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[163708, 153570, 163721, 163876, 98254, 131090, 65634, 108723] avg=131072.0000 max=163876 max_rank=3 min=65634 min_rank=6 imbalance_ratio=1.250275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[100039, 98223, 131300, 129387, 131039, 103358, 153255, 201975] avg=131072.0000 max=201975 max_rank=7 min=98223 min_rank=1 imbalance_ratio=1.540947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] avg=131072.0000 max=132383 max_rank=0 min=128716 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=69592 max_load_reduction_pct=0.344557 moved_to_replicas=180837 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[100039, 98223, 131300, 129387, 131039, 103358, 153255, 201975] avg=131072.0000 max=201975 max_rank=7 min=98223 min_rank=1 imbalance_ratio=1.540947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] avg=131072.0000 max=132384 max_rank=1 min=127135 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=89222 max_load_reduction_pct=0.402615 moved_to_replicas=330644 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] avg=131072.0000 max=132383 max_rank=0 min=128716 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=69592 max_load_reduction_pct=0.344557 moved_to_replicas=180837 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[132382, 132384, 127135, 132384, 130742, 132383, 132382, 128784] avg=131072.0000 max=132384 max_rank=1 min=127135 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=89222 max_load_reduction_pct=0.402615 moved_to_replicas=330644 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] avg=131072.0000 max=132383 max_rank=0 min=128465 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31493 max_load_reduction_pct=0.192176 moved_to_replicas=231665 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] avg=131072.0000 max=132383 max_rank=0 min=128716 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=69592 max_load_reduction_pct=0.344557 moved_to_replicas=180837 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] avg=131072.0000 max=132383 max_rank=0 min=128716 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=69592 max_load_reduction_pct=0.344557 moved_to_replicas=180837 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[100039, 98223, 131300, 129387, 131039, 103358, 153255, 201975] avg=131072.0000 max=201975 max_rank=7 min=98223 min_rank=1 imbalance_ratio=1.540947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[145866, 98383, 187640, 119046, 130997, 233427, 98229, 34988] avg=131072.0000 max=233427 max_rank=5 min=34988 min_rank=7 imbalance_ratio=1.780907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[145866, 98383, 187640, 119046, 130997, 233427, 98229, 34988] avg=131072.0000 max=233427 max_rank=5 min=34988 min_rank=7 imbalance_ratio=1.780907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[163708, 153570, 163721, 163876, 98254, 131090, 65634, 108723] avg=131072.0000 max=163876 max_rank=3 min=65634 min_rank=6 imbalance_ratio=1.250275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] avg=131072.0000 max=132383 max_rank=0 min=128716 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=69592 max_load_reduction_pct=0.344557 moved_to_replicas=180837 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[163708, 153570, 163721, 163876, 98254, 131090, 65634, 108723] avg=131072.0000 max=163876 max_rank=3 min=65634 min_rank=6 imbalance_ratio=1.250275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[100039, 98223, 131300, 129387, 131039, 103358, 153255, 201975] avg=131072.0000 max=201975 max_rank=7 min=98223 min_rank=1 imbalance_ratio=1.540947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[145866, 98383, 187640, 119046, 130997, 233427, 98229, 34988] avg=131072.0000 max=233427 max_rank=5 min=34988 min_rank=7 imbalance_ratio=1.780907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] avg=131072.0000 max=132384 max_rank=3 min=124672 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=101043 max_load_reduction_pct=0.432868 moved_to_replicas=344399 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[145866, 98383, 187640, 119046, 130997, 233427, 98229, 34988] avg=131072.0000 max=233427 max_rank=5 min=34988 min_rank=7 imbalance_ratio=1.780907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] avg=131072.0000 max=132384 max_rank=3 min=124672 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=101043 max_load_reduction_pct=0.432868 moved_to_replicas=344399 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] avg=131072.0000 max=132383 max_rank=0 min=128465 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31493 max_load_reduction_pct=0.192176 moved_to_replicas=231665 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129579, 131090, 128465, 129911] avg=131072.0000 max=132383 max_rank=0 min=128465 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=31493 max_load_reduction_pct=0.192176 moved_to_replicas=231665 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] avg=131072.0000 max=132383 max_rank=0 min=128716 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=69592 max_load_reduction_pct=0.344557 moved_to_replicas=180837 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] avg=131072.0000 max=132384 max_rank=3 min=124672 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=101043 max_load_reduction_pct=0.432868 moved_to_replicas=344399 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] avg=131072.0000 max=132384 max_rank=3 min=124672 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=101043 max_load_reduction_pct=0.432868 moved_to_replicas=344399 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[145866, 98383, 187640, 119046, 130997, 233427, 98229, 34988] avg=131072.0000 max=233427 max_rank=5 min=34988 min_rank=7 imbalance_ratio=1.780907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[65619, 98664, 130668, 212430, 131034, 246250, 98402, 65509] avg=131072.0000 max=246250 max_rank=5 min=65509 min_rank=7 imbalance_ratio=1.878738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[65619, 98664, 130668, 212430, 131034, 246250, 98402, 65509] avg=131072.0000 max=246250 max_rank=5 min=65509 min_rank=7 imbalance_ratio=1.878738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] avg=131072.0000 max=132384 max_rank=3 min=124672 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=101043 max_load_reduction_pct=0.432868 moved_to_replicas=344399 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[100039, 98223, 131300, 129387, 131039, 103358, 153255, 201975] avg=131072.0000 max=201975 max_rank=7 min=98223 min_rank=1 imbalance_ratio=1.540947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[65619, 98664, 130668, 212430, 131034, 246250, 98402, 65509] avg=131072.0000 max=246250 max_rank=5 min=65509 min_rank=7 imbalance_ratio=1.878738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[100039, 98223, 131300, 129387, 131039, 103358, 153255, 201975] avg=131072.0000 max=201975 max_rank=7 min=98223 min_rank=1 imbalance_ratio=1.540947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[145866, 98383, 187640, 119046, 130997, 233427, 98229, 34988] avg=131072.0000 max=233427 max_rank=5 min=34988 min_rank=7 imbalance_ratio=1.780907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] avg=131072.0000 max=132383 max_rank=3 min=128481 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=113867 max_load_reduction_pct=0.462404 moved_to_replicas=387951 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[65619, 98664, 130668, 212430, 131034, 246250, 98402, 65509] avg=131072.0000 max=246250 max_rank=5 min=65509 min_rank=7 imbalance_ratio=1.878738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] avg=131072.0000 max=132383 max_rank=3 min=128481 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=113867 max_load_reduction_pct=0.462404 moved_to_replicas=387951 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] avg=131072.0000 max=132383 max_rank=0 min=128716 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=69592 max_load_reduction_pct=0.344557 moved_to_replicas=180837 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] avg=131072.0000 max=132383 max_rank=3 min=128481 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=113867 max_load_reduction_pct=0.462404 moved_to_replicas=387951 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] avg=131072.0000 max=132384 max_rank=3 min=124672 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=101043 max_load_reduction_pct=0.432868 moved_to_replicas=344399 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[132383, 130987, 131300, 129387, 131039, 128716, 132383, 132381] avg=131072.0000 max=132383 max_rank=0 min=128716 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=69592 max_load_reduction_pct=0.344557 moved_to_replicas=180837 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[65619, 98664, 130668, 212430, 131034, 246250, 98402, 65509] avg=131072.0000 max=246250 max_rank=5 min=65509 min_rank=7 imbalance_ratio=1.878738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] avg=131072.0000 max=132383 max_rank=3 min=128481 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=113867 max_load_reduction_pct=0.462404 moved_to_replicas=387951 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[99150, 106644, 203053, 98339, 98414, 156140, 105344, 181492] avg=131072.0000 max=203053 max_rank=2 min=98339 min_rank=3 imbalance_ratio=1.549171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[99150, 106644, 203053, 98339, 98414, 156140, 105344, 181492] avg=131072.0000 max=203053 max_rank=2 min=98339 min_rank=3 imbalance_ratio=1.549171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] avg=131072.0000 max=132383 max_rank=3 min=128481 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=113867 max_load_reduction_pct=0.462404 moved_to_replicas=387951 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[145866, 98383, 187640, 119046, 130997, 233427, 98229, 34988] avg=131072.0000 max=233427 max_rank=5 min=34988 min_rank=7 imbalance_ratio=1.780907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[99150, 106644, 203053, 98339, 98414, 156140, 105344, 181492] avg=131072.0000 max=203053 max_rank=2 min=98339 min_rank=3 imbalance_ratio=1.549171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] avg=131072.0000 max=132387 max_rank=7 min=126362 min_rank=6 imbalance_ratio=1.010033 max_load_reduction=70666 max_load_reduction_pct=0.348018 moved_to_replicas=281183 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] avg=131072.0000 max=132387 max_rank=7 min=126362 min_rank=6 imbalance_ratio=1.010033 max_load_reduction=70666 max_load_reduction_pct=0.348018 moved_to_replicas=281183 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[65619, 98664, 130668, 212430, 131034, 246250, 98402, 65509] avg=131072.0000 max=246250 max_rank=5 min=65509 min_rank=7 imbalance_ratio=1.878738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[145866, 98383, 187640, 119046, 130997, 233427, 98229, 34988] avg=131072.0000 max=233427 max_rank=5 min=34988 min_rank=7 imbalance_ratio=1.780907 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[99150, 106644, 203053, 98339, 98414, 156140, 105344, 181492] avg=131072.0000 max=203053 max_rank=2 min=98339 min_rank=3 imbalance_ratio=1.549171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] avg=131072.0000 max=132387 max_rank=7 min=126362 min_rank=6 imbalance_ratio=1.010033 max_load_reduction=70666 max_load_reduction_pct=0.348018 moved_to_replicas=281183 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] avg=131072.0000 max=132384 max_rank=3 min=124672 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=101043 max_load_reduction_pct=0.432868 moved_to_replicas=344399 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[99150, 106644, 203053, 98339, 98414, 156140, 105344, 181492] avg=131072.0000 max=203053 max_rank=2 min=98339 min_rank=3 imbalance_ratio=1.549171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] avg=131072.0000 max=132383 max_rank=3 min=128481 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=113867 max_load_reduction_pct=0.462404 moved_to_replicas=387951 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] avg=131072.0000 max=132387 max_rank=7 min=126362 min_rank=6 imbalance_ratio=1.010033 max_load_reduction=70666 max_load_reduction_pct=0.348018 moved_to_replicas=281183 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[132382, 124672, 132383, 132384, 130997, 132383, 130992, 132383] avg=131072.0000 max=132384 max_rank=3 min=124672 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=101043 max_load_reduction_pct=0.432868 moved_to_replicas=344399 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[98453, 130982, 65765, 163585, 98532, 98087, 196578, 196594] avg=131072.0000 max=196594 max_rank=7 min=65765 min_rank=2 imbalance_ratio=1.499893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[98453, 130982, 65765, 163585, 98532, 98087, 196578, 196594] avg=131072.0000 max=196594 max_rank=7 min=65765 min_rank=2 imbalance_ratio=1.499893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] avg=131072.0000 max=132387 max_rank=7 min=126362 min_rank=6 imbalance_ratio=1.010033 max_load_reduction=70666 max_load_reduction_pct=0.348018 moved_to_replicas=281183 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[98453, 130982, 65765, 163585, 98532, 98087, 196578, 196594] avg=131072.0000 max=196594 max_rank=7 min=65765 min_rank=2 imbalance_ratio=1.499893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[65619, 98664, 130668, 212430, 131034, 246250, 98402, 65509] avg=131072.0000 max=246250 max_rank=5 min=65509 min_rank=7 imbalance_ratio=1.878738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129534 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64211 max_load_reduction_pct=0.326617 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129534 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64211 max_load_reduction_pct=0.326617 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[99150, 106644, 203053, 98339, 98414, 156140, 105344, 181492] avg=131072.0000 max=203053 max_rank=2 min=98339 min_rank=3 imbalance_ratio=1.549171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[98453, 130982, 65765, 163585, 98532, 98087, 196578, 196594] avg=131072.0000 max=196594 max_rank=7 min=65765 min_rank=2 imbalance_ratio=1.499893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[65619, 98664, 130668, 212430, 131034, 246250, 98402, 65509] avg=131072.0000 max=246250 max_rank=5 min=65509 min_rank=7 imbalance_ratio=1.878738 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129534 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64211 max_load_reduction_pct=0.326617 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[98453, 130982, 65765, 163585, 98532, 98087, 196578, 196594] avg=131072.0000 max=196594 max_rank=7 min=65765 min_rank=2 imbalance_ratio=1.499893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] avg=131072.0000 max=132383 max_rank=3 min=128481 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=113867 max_load_reduction_pct=0.462404 moved_to_replicas=387951 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129534 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64211 max_load_reduction_pct=0.326617 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] avg=131072.0000 max=132387 max_rank=7 min=126362 min_rank=6 imbalance_ratio=1.010033 max_load_reduction=70666 max_load_reduction_pct=0.348018 moved_to_replicas=281183 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[128481, 131426, 130668, 132383, 131034, 132383, 131165, 131036] avg=131072.0000 max=132383 max_rank=3 min=128481 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=113867 max_load_reduction_pct=0.462404 moved_to_replicas=387951 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[123606, 94337, 143668, 98448, 64014, 163802, 131057, 229644] avg=131072.0000 max=229644 max_rank=7 min=64014 min_rank=4 imbalance_ratio=1.752045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[123606, 94337, 143668, 98448, 64014, 163802, 131057, 229644] avg=131072.0000 max=229644 max_rank=7 min=64014 min_rank=4 imbalance_ratio=1.752045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129534 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64211 max_load_reduction_pct=0.326617 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[123606, 94337, 143668, 98448, 64014, 163802, 131057, 229644] avg=131072.0000 max=229644 max_rank=7 min=64014 min_rank=4 imbalance_ratio=1.752045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[99150, 106644, 203053, 98339, 98414, 156140, 105344, 181492] avg=131072.0000 max=203053 max_rank=2 min=98339 min_rank=3 imbalance_ratio=1.549171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] avg=131072.0000 max=132383 max_rank=0 min=128512 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=97261 max_load_reduction_pct=0.423529 moved_to_replicas=280297 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[123606, 94337, 143668, 98448, 64014, 163802, 131057, 229644] avg=131072.0000 max=229644 max_rank=7 min=64014 min_rank=4 imbalance_ratio=1.752045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[98453, 130982, 65765, 163585, 98532, 98087, 196578, 196594] avg=131072.0000 max=196594 max_rank=7 min=65765 min_rank=2 imbalance_ratio=1.499893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] avg=131072.0000 max=132383 max_rank=0 min=128512 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=97261 max_load_reduction_pct=0.423529 moved_to_replicas=280297 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] avg=131072.0000 max=132383 max_rank=0 min=128512 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=97261 max_load_reduction_pct=0.423529 moved_to_replicas=280297 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[99150, 106644, 203053, 98339, 98414, 156140, 105344, 181492] avg=131072.0000 max=203053 max_rank=2 min=98339 min_rank=3 imbalance_ratio=1.549171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[123606, 94337, 143668, 98448, 64014, 163802, 131057, 229644] avg=131072.0000 max=229644 max_rank=7 min=64014 min_rank=4 imbalance_ratio=1.752045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] avg=131072.0000 max=132387 max_rank=7 min=126362 min_rank=6 imbalance_ratio=1.010033 max_load_reduction=70666 max_load_reduction_pct=0.348018 moved_to_replicas=281183 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] avg=131072.0000 max=132383 max_rank=0 min=128512 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=97261 max_load_reduction_pct=0.423529 moved_to_replicas=280297 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129534 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64211 max_load_reduction_pct=0.326617 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[35574, 196608, 32975, 195793, 98541, 259706, 65571, 163808] avg=131072.0000 max=259706 max_rank=5 min=32975 min_rank=2 imbalance_ratio=1.981400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] avg=131072.0000 max=132383 max_rank=0 min=128512 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=97261 max_load_reduction_pct=0.423529 moved_to_replicas=280297 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[132382, 130400, 132382, 131103, 131176, 132384, 126362, 132387] avg=131072.0000 max=132387 max_rank=7 min=126362 min_rank=6 imbalance_ratio=1.010033 max_load_reduction=70666 max_load_reduction_pct=0.348018 moved_to_replicas=281183 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[35574, 196608, 32975, 195793, 98541, 259706, 65571, 163808] avg=131072.0000 max=259706 max_rank=5 min=32975 min_rank=2 imbalance_ratio=1.981400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[35574, 196608, 32975, 195793, 98541, 259706, 65571, 163808] avg=131072.0000 max=259706 max_rank=5 min=32975 min_rank=2 imbalance_ratio=1.981400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] avg=131072.0000 max=132383 max_rank=0 min=126379 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=127323 max_load_reduction_pct=0.490258 moved_to_replicas=571208 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[98453, 130982, 65765, 163585, 98532, 98087, 196578, 196594] avg=131072.0000 max=196594 max_rank=7 min=65765 min_rank=2 imbalance_ratio=1.499893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[35574, 196608, 32975, 195793, 98541, 259706, 65571, 163808] avg=131072.0000 max=259706 max_rank=5 min=32975 min_rank=2 imbalance_ratio=1.981400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] avg=131072.0000 max=132383 max_rank=0 min=126379 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=127323 max_load_reduction_pct=0.490258 moved_to_replicas=571208 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] avg=131072.0000 max=132383 max_rank=0 min=126379 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=127323 max_load_reduction_pct=0.490258 moved_to_replicas=571208 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[123606, 94337, 143668, 98448, 64014, 163802, 131057, 229644] avg=131072.0000 max=229644 max_rank=7 min=64014 min_rank=4 imbalance_ratio=1.752045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[35574, 196608, 32975, 195793, 98541, 259706, 65571, 163808] avg=131072.0000 max=259706 max_rank=5 min=32975 min_rank=2 imbalance_ratio=1.981400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[98453, 130982, 65765, 163585, 98532, 98087, 196578, 196594] avg=131072.0000 max=196594 max_rank=7 min=65765 min_rank=2 imbalance_ratio=1.499893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129534 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64211 max_load_reduction_pct=0.326617 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] avg=131072.0000 max=132383 max_rank=0 min=126379 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=127323 max_load_reduction_pct=0.490258 moved_to_replicas=571208 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[164015, 98883, 64850, 163619, 130959, 165147, 163767, 97336] avg=131072.0000 max=165147 max_rank=5 min=64850 min_rank=2 imbalance_ratio=1.259972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] avg=131072.0000 max=132383 max_rank=0 min=128512 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=97261 max_load_reduction_pct=0.423529 moved_to_replicas=280297 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] avg=131072.0000 max=132383 max_rank=0 min=126379 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=127323 max_load_reduction_pct=0.490258 moved_to_replicas=571208 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[131217, 130982, 129960, 132383, 129734, 129534, 132383, 132383] avg=131072.0000 max=132383 max_rank=3 min=129534 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=64211 max_load_reduction_pct=0.326617 moved_to_replicas=319501 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[164015, 98883, 64850, 163619, 130959, 165147, 163767, 97336] avg=131072.0000 max=165147 max_rank=5 min=64850 min_rank=2 imbalance_ratio=1.259972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[164015, 98883, 64850, 163619, 130959, 165147, 163767, 97336] avg=131072.0000 max=165147 max_rank=5 min=64850 min_rank=2 imbalance_ratio=1.259972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] avg=131072.0000 max=132383 max_rank=0 min=128969 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32764 max_load_reduction_pct=0.198393 moved_to_replicas=253305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[123606, 94337, 143668, 98448, 64014, 163802, 131057, 229644] avg=131072.0000 max=229644 max_rank=7 min=64014 min_rank=4 imbalance_ratio=1.752045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[164015, 98883, 64850, 163619, 130959, 165147, 163767, 97336] avg=131072.0000 max=165147 max_rank=5 min=64850 min_rank=2 imbalance_ratio=1.259972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] avg=131072.0000 max=132383 max_rank=0 min=128969 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32764 max_load_reduction_pct=0.198393 moved_to_replicas=253305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[164015, 98883, 64850, 163619, 130959, 165147, 163767, 97336] avg=131072.0000 max=165147 max_rank=5 min=64850 min_rank=2 imbalance_ratio=1.259972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] avg=131072.0000 max=132383 max_rank=0 min=128969 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32764 max_load_reduction_pct=0.198393 moved_to_replicas=253305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[35574, 196608, 32975, 195793, 98541, 259706, 65571, 163808] avg=131072.0000 max=259706 max_rank=5 min=32975 min_rank=2 imbalance_ratio=1.981400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[123606, 94337, 143668, 98448, 64014, 163802, 131057, 229644] avg=131072.0000 max=229644 max_rank=7 min=64014 min_rank=4 imbalance_ratio=1.752045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] avg=131072.0000 max=132383 max_rank=0 min=128512 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=97261 max_load_reduction_pct=0.423529 moved_to_replicas=280297 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] avg=131072.0000 max=132383 max_rank=0 min=128969 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32764 max_load_reduction_pct=0.198393 moved_to_replicas=253305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[196762, 196587, 65573, 163666, 98301, 65794, 131116, 130777] avg=131072.0000 max=196762 max_rank=0 min=65573 min_rank=2 imbalance_ratio=1.501175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] avg=131072.0000 max=132383 max_rank=0 min=128969 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32764 max_load_reduction_pct=0.198393 moved_to_replicas=253305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] avg=131072.0000 max=132383 max_rank=0 min=126379 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=127323 max_load_reduction_pct=0.490258 moved_to_replicas=571208 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[196762, 196587, 65573, 163666, 98301, 65794, 131116, 130777] avg=131072.0000 max=196762 max_rank=0 min=65573 min_rank=2 imbalance_ratio=1.501175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[132383, 129608, 132383, 129867, 128512, 132383, 131057, 132383] avg=131072.0000 max=132383 max_rank=0 min=128512 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=97261 max_load_reduction_pct=0.423529 moved_to_replicas=280297 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[196762, 196587, 65573, 163666, 98301, 65794, 131116, 130777] avg=131072.0000 max=196762 max_rank=0 min=65573 min_rank=2 imbalance_ratio=1.501175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] avg=131072.0000 max=132383 max_rank=0 min=129618 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64379 max_load_reduction_pct=0.327192 moved_to_replicas=319954 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[35574, 196608, 32975, 195793, 98541, 259706, 65571, 163808] avg=131072.0000 max=259706 max_rank=5 min=32975 min_rank=2 imbalance_ratio=1.981400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[196762, 196587, 65573, 163666, 98301, 65794, 131116, 130777] avg=131072.0000 max=196762 max_rank=0 min=65573 min_rank=2 imbalance_ratio=1.501175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] avg=131072.0000 max=132383 max_rank=0 min=129618 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64379 max_load_reduction_pct=0.327192 moved_to_replicas=319954 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] avg=131072.0000 max=132383 max_rank=0 min=129618 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64379 max_load_reduction_pct=0.327192 moved_to_replicas=319954 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[196762, 196587, 65573, 163666, 98301, 65794, 131116, 130777] avg=131072.0000 max=196762 max_rank=0 min=65573 min_rank=2 imbalance_ratio=1.501175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[164015, 98883, 64850, 163619, 130959, 165147, 163767, 97336] avg=131072.0000 max=165147 max_rank=5 min=64850 min_rank=2 imbalance_ratio=1.259972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[35574, 196608, 32975, 195793, 98541, 259706, 65571, 163808] avg=131072.0000 max=259706 max_rank=5 min=32975 min_rank=2 imbalance_ratio=1.981400 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] avg=131072.0000 max=132383 max_rank=0 min=126379 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=127323 max_load_reduction_pct=0.490258 moved_to_replicas=571208 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] avg=131072.0000 max=132383 max_rank=0 min=129618 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64379 max_load_reduction_pct=0.327192 moved_to_replicas=319954 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[65648, 165454, 127806, 165153, 98250, 196557, 163829, 65879] avg=131072.0000 max=196557 max_rank=5 min=65648 min_rank=0 imbalance_ratio=1.499611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] avg=131072.0000 max=132383 max_rank=0 min=129618 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64379 max_load_reduction_pct=0.327192 moved_to_replicas=319954 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] avg=131072.0000 max=132383 max_rank=0 min=128969 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32764 max_load_reduction_pct=0.198393 moved_to_replicas=253305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[65648, 165454, 127806, 165153, 98250, 196557, 163829, 65879] avg=131072.0000 max=196557 max_rank=5 min=65648 min_rank=0 imbalance_ratio=1.499611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[65648, 165454, 127806, 165153, 98250, 196557, 163829, 65879] avg=131072.0000 max=196557 max_rank=5 min=65648 min_rank=0 imbalance_ratio=1.499611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[132383, 132383, 126379, 132383, 129187, 132383, 131095, 132383] avg=131072.0000 max=132383 max_rank=0 min=126379 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=127323 max_load_reduction_pct=0.490258 moved_to_replicas=571208 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] avg=131072.0000 max=132383 max_rank=1 min=127806 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64174 max_load_reduction_pct=0.326491 moved_to_replicas=323190 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] avg=131072.0000 max=132383 max_rank=1 min=127806 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64174 max_load_reduction_pct=0.326491 moved_to_replicas=323190 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[164015, 98883, 64850, 163619, 130959, 165147, 163767, 97336] avg=131072.0000 max=165147 max_rank=5 min=64850 min_rank=2 imbalance_ratio=1.259972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[65648, 165454, 127806, 165153, 98250, 196557, 163829, 65879] avg=131072.0000 max=196557 max_rank=5 min=65648 min_rank=0 imbalance_ratio=1.499611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] avg=131072.0000 max=132383 max_rank=1 min=127806 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64174 max_load_reduction_pct=0.326491 moved_to_replicas=323190 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[65648, 165454, 127806, 165153, 98250, 196557, 163829, 65879] avg=131072.0000 max=196557 max_rank=5 min=65648 min_rank=0 imbalance_ratio=1.499611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[196762, 196587, 65573, 163666, 98301, 65794, 131116, 130777] avg=131072.0000 max=196762 max_rank=0 min=65573 min_rank=2 imbalance_ratio=1.501175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[164015, 98883, 64850, 163619, 130959, 165147, 163767, 97336] avg=131072.0000 max=165147 max_rank=5 min=64850 min_rank=2 imbalance_ratio=1.259972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[131196, 98491, 131080, 98051, 131134, 163777, 229289, 65558] avg=131072.0000 max=229289 max_rank=6 min=65558 min_rank=7 imbalance_ratio=1.749336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] avg=131072.0000 max=132383 max_rank=0 min=128969 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32764 max_load_reduction_pct=0.198393 moved_to_replicas=253305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] avg=131072.0000 max=132383 max_rank=1 min=127806 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64174 max_load_reduction_pct=0.326491 moved_to_replicas=323190 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] avg=131072.0000 max=132383 max_rank=1 min=127806 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64174 max_load_reduction_pct=0.326491 moved_to_replicas=323190 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[131196, 98491, 131080, 98051, 131134, 163777, 229289, 65558] avg=131072.0000 max=229289 max_rank=6 min=65558 min_rank=7 imbalance_ratio=1.749336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] avg=131072.0000 max=132383 max_rank=0 min=129618 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64379 max_load_reduction_pct=0.327192 moved_to_replicas=319954 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[131196, 98491, 131080, 98051, 131134, 163777, 229289, 65558] avg=131072.0000 max=229289 max_rank=6 min=65558 min_rank=7 imbalance_ratio=1.749336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[132383, 130119, 128997, 132383, 130959, 132383, 132383, 128969] avg=131072.0000 max=132383 max_rank=0 min=128969 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=32764 max_load_reduction_pct=0.198393 moved_to_replicas=253305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] avg=131072.0000 max=132383 max_rank=5 min=129701 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96906 max_load_reduction_pct=0.422637 moved_to_replicas=256719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] avg=131072.0000 max=132383 max_rank=5 min=129701 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96906 max_load_reduction_pct=0.422637 moved_to_replicas=256719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[196762, 196587, 65573, 163666, 98301, 65794, 131116, 130777] avg=131072.0000 max=196762 max_rank=0 min=65573 min_rank=2 imbalance_ratio=1.501175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] avg=131072.0000 max=132383 max_rank=5 min=129701 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96906 max_load_reduction_pct=0.422637 moved_to_replicas=256719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[131196, 98491, 131080, 98051, 131134, 163777, 229289, 65558] avg=131072.0000 max=229289 max_rank=6 min=65558 min_rank=7 imbalance_ratio=1.749336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[131196, 98491, 131080, 98051, 131134, 163777, 229289, 65558] avg=131072.0000 max=229289 max_rank=6 min=65558 min_rank=7 imbalance_ratio=1.749336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[65648, 165454, 127806, 165153, 98250, 196557, 163829, 65879] avg=131072.0000 max=196557 max_rank=5 min=65648 min_rank=0 imbalance_ratio=1.499611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[33494, 163843, 65591, 132471, 96126, 229150, 131098, 196803] avg=131072.0000 max=229150 max_rank=5 min=33494 min_rank=0 imbalance_ratio=1.748276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[196762, 196587, 65573, 163666, 98301, 65794, 131116, 130777] avg=131072.0000 max=196762 max_rank=0 min=65573 min_rank=2 imbalance_ratio=1.501175 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] avg=131072.0000 max=132383 max_rank=0 min=129618 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64379 max_load_reduction_pct=0.327192 moved_to_replicas=319954 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] avg=131072.0000 max=132383 max_rank=5 min=129701 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96906 max_load_reduction_pct=0.422637 moved_to_replicas=256719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[33494, 163843, 65591, 132471, 96126, 229150, 131098, 196803] avg=131072.0000 max=229150 max_rank=5 min=33494 min_rank=0 imbalance_ratio=1.748276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] avg=131072.0000 max=132383 max_rank=5 min=129701 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96906 max_load_reduction_pct=0.422637 moved_to_replicas=256719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] avg=131072.0000 max=132383 max_rank=1 min=127806 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64174 max_load_reduction_pct=0.326491 moved_to_replicas=323190 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[33494, 163843, 65591, 132471, 96126, 229150, 131098, 196803] avg=131072.0000 max=229150 max_rank=5 min=33494 min_rank=0 imbalance_ratio=1.748276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] avg=131072.0000 max=132383 max_rank=1 min=128977 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96767 max_load_reduction_pct=0.422287 moved_to_replicas=385439 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[132383, 132383, 129618, 132383, 129742, 130174, 131116, 130777] avg=131072.0000 max=132383 max_rank=0 min=129618 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64379 max_load_reduction_pct=0.327192 moved_to_replicas=319954 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] avg=131072.0000 max=132383 max_rank=1 min=128977 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96767 max_load_reduction_pct=0.422287 moved_to_replicas=385439 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[65648, 165454, 127806, 165153, 98250, 196557, 163829, 65879] avg=131072.0000 max=196557 max_rank=5 min=65648 min_rank=0 imbalance_ratio=1.499611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] avg=131072.0000 max=132383 max_rank=1 min=128977 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96767 max_load_reduction_pct=0.422287 moved_to_replicas=385439 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[33494, 163843, 65591, 132471, 96126, 229150, 131098, 196803] avg=131072.0000 max=229150 max_rank=5 min=33494 min_rank=0 imbalance_ratio=1.748276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[33494, 163843, 65591, 132471, 96126, 229150, 131098, 196803] avg=131072.0000 max=229150 max_rank=5 min=33494 min_rank=0 imbalance_ratio=1.748276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[131196, 98491, 131080, 98051, 131134, 163777, 229289, 65558] avg=131072.0000 max=229289 max_rank=6 min=65558 min_rank=7 imbalance_ratio=1.749336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[65648, 165454, 127806, 165153, 98250, 196557, 163829, 65879] avg=131072.0000 max=196557 max_rank=5 min=65648 min_rank=0 imbalance_ratio=1.499611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] avg=131072.0000 max=132383 max_rank=1 min=127806 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64174 max_load_reduction_pct=0.326491 moved_to_replicas=323190 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] avg=131072.0000 max=132383 max_rank=1 min=128977 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96767 max_load_reduction_pct=0.422287 moved_to_replicas=385439 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] avg=131072.0000 max=132383 max_rank=1 min=128977 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96767 max_load_reduction_pct=0.422287 moved_to_replicas=385439 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] avg=131072.0000 max=132383 max_rank=5 min=129701 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96906 max_load_reduction_pct=0.422637 moved_to_replicas=256719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[131175, 132383, 127806, 132383, 130010, 132383, 132383, 130053] avg=131072.0000 max=132383 max_rank=1 min=127806 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64174 max_load_reduction_pct=0.326491 moved_to_replicas=323190 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[131196, 98491, 131080, 98051, 131134, 163777, 229289, 65558] avg=131072.0000 max=229289 max_rank=6 min=65558 min_rank=7 imbalance_ratio=1.749336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[33494, 163843, 65591, 132471, 96126, 229150, 131098, 196803] avg=131072.0000 max=229150 max_rank=5 min=33494 min_rank=0 imbalance_ratio=1.748276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] avg=131072.0000 max=132383 max_rank=5 min=129701 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96906 max_load_reduction_pct=0.422637 moved_to_replicas=256719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[131196, 98491, 131080, 98051, 131134, 163777, 229289, 65558] avg=131072.0000 max=229289 max_rank=6 min=65558 min_rank=7 imbalance_ratio=1.749336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] avg=131072.0000 max=132383 max_rank=1 min=128977 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96767 max_load_reduction_pct=0.422287 moved_to_replicas=385439 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[131196, 129885, 131080, 130814, 131134, 132383, 132383, 129701] avg=131072.0000 max=132383 max_rank=5 min=129701 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=96906 max_load_reduction_pct=0.422637 moved_to_replicas=256719 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[33494, 163843, 65591, 132471, 96126, 229150, 131098, 196803] avg=131072.0000 max=229150 max_rank=5 min=33494 min_rank=0 imbalance_ratio=1.748276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] avg=131072.0000 max=132383 max_rank=1 min=128977 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96767 max_load_reduction_pct=0.422287 moved_to_replicas=385439 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[33494, 163843, 65591, 132471, 96126, 229150, 131098, 196803] avg=131072.0000 max=229150 max_rank=5 min=33494 min_rank=0 imbalance_ratio=1.748276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[129168, 132383, 129801, 132383, 128977, 132383, 131098, 132383] avg=131072.0000 max=132383 max_rank=1 min=128977 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96767 max_load_reduction_pct=0.422287 moved_to_replicas=385439 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=4213331 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=996224 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=4149461 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1058560 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=4212093 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=996096 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=4209324 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1000576 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=4204103 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1007488 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=4193879 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015552 peak_demand=132387 required_capacity=1.010033 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=4195193 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014784 peak_demand=132383 required_capacity=1.010002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=4177048 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1031808 peak_demand=132384 required_capacity=1.010010 + [2026-08-30 05:17:56] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 793.1 | throughput per GPU (TFLOP/s/GPU): 420.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.071848E+00 | loss scale: 1.0 | grad norm: 5.395 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=163840 expected_recv=259519 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=163840 expected_recv=261751 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=163840 expected_recv=255387 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=163840 expected_recv=261433 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=163840 expected_recv=259519 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=163840 expected_recv=261751 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=163840 expected_recv=255387 actual_valid_recv=257792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=163840 expected_recv=261433 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=257792 slack_slots=69888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=163840 expected_recv=256880 expected_overflow=0 exact_recv_per_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=163840 expected_recv=264460 expected_overflow=0 exact_recv_per_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=163840 expected_recv=257215 expected_overflow=0 exact_recv_per_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=163840 expected_recv=259533 expected_overflow=0 exact_recv_per_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=163840 expected_recv=256880 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=163840 expected_recv=264460 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=163840 expected_recv=257215 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=259968 slack_slots=67712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=163840 expected_recv=259533 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=259840 slack_slots=67840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=163840 expected_recv=257467 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=163840 expected_recv=255565 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=163840 expected_recv=257467 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=163840 expected_recv=255565 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=163840 expected_recv=260289 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=259584 slack_slots=68096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=257408 slack_slots=70272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=163840 expected_recv=260289 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=163840 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=163840 expected_recv=264763 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=163840 expected_recv=263816 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=163840 expected_recv=261465 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=163840 expected_recv=254648 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=163840 expected_recv=263816 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=163840 expected_recv=261465 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=163840 expected_recv=254648 actual_valid_recv=256768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=163840 expected_recv=258160 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=163840 expected_recv=261911 expected_overflow=0 exact_recv_per_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=256768 slack_slots=70912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=163840 expected_recv=258160 actual_valid_recv=259840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=163840 expected_recv=261911 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=259840 slack_slots=67840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=163840 expected_recv=263601 expected_overflow=0 exact_recv_per_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=163840 expected_recv=253076 expected_overflow=0 exact_recv_per_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=163840 expected_recv=263601 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=163840 expected_recv=253076 actual_valid_recv=254592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=265856 slack_slots=61824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=254592 slack_slots=73088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262135 expected_overflow=0 exact_recv_per_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262135 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262131 expected_overflow=0 exact_recv_per_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262681 expected_overflow=0 exact_recv_per_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262131 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262681 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=163840 expected_recv=251025 expected_overflow=0 exact_recv_per_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=163840 expected_recv=251025 actual_valid_recv=252672 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=252672 slack_slots=75008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=163840 expected_recv=264761 expected_overflow=0 exact_recv_per_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262267 expected_overflow=0 exact_recv_per_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262117 expected_overflow=0 exact_recv_per_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=163840 expected_recv=264761 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262267 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262117 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=163840 expected_recv=258978 expected_overflow=0 exact_recv_per_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=163840 expected_recv=258978 actual_valid_recv=260608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=163840 expected_recv=257985 expected_overflow=0 exact_recv_per_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=260608 slack_slots=67072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=163840 expected_recv=257985 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=259712 slack_slots=67968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=163840 expected_recv=258501 expected_overflow=0 exact_recv_per_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=163840 expected_recv=258501 actual_valid_recv=259968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262621 expected_overflow=0 exact_recv_per_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=259968 slack_slots=67712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=163840 expected_recv=259686 expected_overflow=0 exact_recv_per_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262621 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=163840 expected_recv=254905 expected_overflow=0 exact_recv_per_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=163840 expected_recv=259686 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=163840 expected_recv=254905 actual_valid_recv=256128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262547 expected_overflow=0 exact_recv_per_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=256128 slack_slots=71552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262547 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=163840 expected_recv=263039 expected_overflow=0 exact_recv_per_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=163840 expected_recv=263948 expected_overflow=0 exact_recv_per_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=163840 expected_recv=263039 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=163840 expected_recv=261769 expected_overflow=0 exact_recv_per_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=163840 expected_recv=263948 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=163840 expected_recv=258544 expected_overflow=0 exact_recv_per_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265472 slack_slots=62208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=163840 expected_recv=261769 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261910 expected_overflow=0 exact_recv_per_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=163840 expected_recv=258544 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261910 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=259584 slack_slots=68096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261962 expected_overflow=0 exact_recv_per_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261962 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=163840 expected_recv=257399 expected_overflow=0 exact_recv_per_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262836 expected_overflow=0 exact_recv_per_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=163840 expected_recv=257399 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=163840 expected_recv=248580 expected_overflow=0 exact_recv_per_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262836 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=259072 slack_slots=68608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=163840 expected_recv=248580 actual_valid_recv=250112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=250112 slack_slots=77568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=163840 expected_recv=261907 expected_overflow=0 exact_recv_per_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=163840 expected_recv=261907 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=163840 expected_recv=250458 expected_overflow=0 exact_recv_per_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=163840 expected_recv=258097 expected_overflow=0 exact_recv_per_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=163840 expected_recv=250458 actual_valid_recv=252160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=163840 expected_recv=258097 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=252160 slack_slots=75520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=259712 slack_slots=67968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264770 expected_overflow=0 exact_recv_per_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264770 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=163840 expected_recv=254957 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=163840 expected_recv=254957 actual_valid_recv=256640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=256640 slack_slots=71040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=163840 expected_recv=259307 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=265856 slack_slots=61824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=163840 expected_recv=259307 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=163840 expected_recv=259362 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=163840 expected_recv=264762 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=260864 slack_slots=66816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=163840 expected_recv=259362 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=163840 expected_recv=259057 expected_overflow=0 exact_recv_per_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=163840 expected_recv=264762 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=260864 slack_slots=66816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=163840 expected_recv=258409 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=163840 expected_recv=259057 actual_valid_recv=260352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=163840 expected_recv=258409 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=260352 slack_slots=67328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=265856 slack_slots=61824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=163840 expected_recv=261121 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262288 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265856 slack_slots=61824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=163840 expected_recv=261121 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262288 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=163840 expected_recv=260489 expected_overflow=0 exact_recv_per_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=163840 expected_recv=259595 expected_overflow=0 exact_recv_per_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=163840 expected_recv=260489 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=163840 expected_recv=261678 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=163840 expected_recv=259595 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=163840 expected_recv=263638 expected_overflow=0 exact_recv_per_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=163840 expected_recv=261678 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=163840 expected_recv=263638 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=163840 expected_recv=259899 expected_overflow=0 exact_recv_per_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=163840 expected_recv=259899 actual_valid_recv=261376 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=163840 expected_recv=257778 expected_overflow=0 exact_recv_per_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=261376 slack_slots=66304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262069 expected_overflow=0 exact_recv_per_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=163840 expected_recv=257778 actual_valid_recv=259200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=163840 expected_recv=258105 expected_overflow=0 exact_recv_per_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=163840 expected_recv=263551 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262069 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=163840 expected_recv=259474 expected_overflow=0 exact_recv_per_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=259200 slack_slots=68480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=163840 expected_recv=258105 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=163840 expected_recv=263551 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=163840 expected_recv=259474 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=259712 slack_slots=67968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=260480 slack_slots=67200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262383 expected_overflow=0 exact_recv_per_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262383 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262215 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262215 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262280 expected_overflow=0 exact_recv_per_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[152274, 141997, 119871, 92842, 148436, 96664, 136451, 160041] avg=131072.0000 max=160041 max_rank=7 min=92842 min_rank=3 imbalance_ratio=1.221016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262280 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266112 slack_slots=61568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262325 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128531 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=27658 max_load_reduction_pct=0.172818 moved_to_replicas=151606 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262325 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[152274, 141997, 119871, 92842, 148436, 96664, 136451, 160041] avg=131072.0000 max=160041 max_rank=7 min=92842 min_rank=3 imbalance_ratio=1.221016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[152274, 141997, 119871, 92842, 148436, 96664, 136451, 160041] avg=131072.0000 max=160041 max_rank=7 min=92842 min_rank=3 imbalance_ratio=1.221016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=163840 expected_recv=249998 expected_overflow=0 exact_recv_per_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[152274, 141997, 119871, 92842, 148436, 96664, 136451, 160041] avg=131072.0000 max=160041 max_rank=7 min=92842 min_rank=3 imbalance_ratio=1.221016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=163840 expected_recv=249998 actual_valid_recv=251648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[152274, 141997, 119871, 92842, 148436, 96664, 136451, 160041] avg=131072.0000 max=160041 max_rank=7 min=92842 min_rank=3 imbalance_ratio=1.221016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128531 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=27658 max_load_reduction_pct=0.172818 moved_to_replicas=151606 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[57887, 132077, 152588, 193931, 159301, 101022, 138426, 113344] avg=131072.0000 max=193931 max_rank=3 min=57887 min_rank=0 imbalance_ratio=1.479576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=251648 slack_slots=76032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128531 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=27658 max_load_reduction_pct=0.172818 moved_to_replicas=151606 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128531 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=27658 max_load_reduction_pct=0.172818 moved_to_replicas=151606 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[152274, 141997, 119871, 92842, 148436, 96664, 136451, 160041] avg=131072.0000 max=160041 max_rank=7 min=92842 min_rank=3 imbalance_ratio=1.221016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[152274, 141997, 119871, 92842, 148436, 96664, 136451, 160041] avg=131072.0000 max=160041 max_rank=7 min=92842 min_rank=3 imbalance_ratio=1.221016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128531 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=27658 max_load_reduction_pct=0.172818 moved_to_replicas=151606 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] avg=131072.0000 max=132385 max_rank=4 min=126642 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=61546 max_load_reduction_pct=0.317360 moved_to_replicas=228758 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[57887, 132077, 152588, 193931, 159301, 101022, 138426, 113344] avg=131072.0000 max=193931 max_rank=3 min=57887 min_rank=0 imbalance_ratio=1.479576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[57887, 132077, 152588, 193931, 159301, 101022, 138426, 113344] avg=131072.0000 max=193931 max_rank=3 min=57887 min_rank=0 imbalance_ratio=1.479576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128531 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=27658 max_load_reduction_pct=0.172818 moved_to_replicas=151606 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128531 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=27658 max_load_reduction_pct=0.172818 moved_to_replicas=151606 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[152274, 141997, 119871, 92842, 148436, 96664, 136451, 160041] avg=131072.0000 max=160041 max_rank=7 min=92842 min_rank=3 imbalance_ratio=1.221016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[57887, 132077, 152588, 193931, 159301, 101022, 138426, 113344] avg=131072.0000 max=193931 max_rank=3 min=57887 min_rank=0 imbalance_ratio=1.479576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] avg=131072.0000 max=132385 max_rank=4 min=126642 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=61546 max_load_reduction_pct=0.317360 moved_to_replicas=228758 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[145773, 167201, 98483, 251327, 130440, 126623, 45500, 83229] avg=131072.0000 max=251327 max_rank=3 min=45500 min_rank=6 imbalance_ratio=1.917473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[57887, 132077, 152588, 193931, 159301, 101022, 138426, 113344] avg=131072.0000 max=193931 max_rank=3 min=57887 min_rank=0 imbalance_ratio=1.479576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] avg=131072.0000 max=132385 max_rank=4 min=126642 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=61546 max_load_reduction_pct=0.317360 moved_to_replicas=228758 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] avg=131072.0000 max=132385 max_rank=4 min=126642 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=61546 max_load_reduction_pct=0.317360 moved_to_replicas=228758 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[132383, 132382, 129486, 128645, 132383, 128531, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=128531 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=27658 max_load_reduction_pct=0.172818 moved_to_replicas=151606 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] avg=131072.0000 max=132384 max_rank=3 min=127383 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=118943 max_load_reduction_pct=0.473260 moved_to_replicas=334983 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[57887, 132077, 152588, 193931, 159301, 101022, 138426, 113344] avg=131072.0000 max=193931 max_rank=3 min=57887 min_rank=0 imbalance_ratio=1.479576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[57887, 132077, 152588, 193931, 159301, 101022, 138426, 113344] avg=131072.0000 max=193931 max_rank=3 min=57887 min_rank=0 imbalance_ratio=1.479576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] avg=131072.0000 max=132385 max_rank=4 min=126642 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=61546 max_load_reduction_pct=0.317360 moved_to_replicas=228758 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[145773, 167201, 98483, 251327, 130440, 126623, 45500, 83229] avg=131072.0000 max=251327 max_rank=3 min=45500 min_rank=6 imbalance_ratio=1.917473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[145773, 167201, 98483, 251327, 130440, 126623, 45500, 83229] avg=131072.0000 max=251327 max_rank=3 min=45500 min_rank=6 imbalance_ratio=1.917473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] avg=131072.0000 max=132385 max_rank=4 min=126642 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=61546 max_load_reduction_pct=0.317360 moved_to_replicas=228758 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[145773, 167201, 98483, 251327, 130440, 126623, 45500, 83229] avg=131072.0000 max=251327 max_rank=3 min=45500 min_rank=6 imbalance_ratio=1.917473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] avg=131072.0000 max=132385 max_rank=4 min=126642 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=61546 max_load_reduction_pct=0.317360 moved_to_replicas=228758 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[57887, 132077, 152588, 193931, 159301, 101022, 138426, 113344] avg=131072.0000 max=193931 max_rank=3 min=57887 min_rank=0 imbalance_ratio=1.479576 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] avg=131072.0000 max=132384 max_rank=3 min=127383 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=118943 max_load_reduction_pct=0.473260 moved_to_replicas=334983 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[162955, 140620, 164443, 163397, 98206, 131433, 83738, 103784] avg=131072.0000 max=164443 max_rank=2 min=83738 min_rank=6 imbalance_ratio=1.254601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] avg=131072.0000 max=132384 max_rank=3 min=127383 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=118943 max_load_reduction_pct=0.473260 moved_to_replicas=334983 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[145773, 167201, 98483, 251327, 130440, 126623, 45500, 83229] avg=131072.0000 max=251327 max_rank=3 min=45500 min_rank=6 imbalance_ratio=1.917473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] avg=131072.0000 max=132384 max_rank=3 min=127383 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=118943 max_load_reduction_pct=0.473260 moved_to_replicas=334983 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[126642, 132077, 132383, 132383, 132385, 127938, 132385, 132383] avg=131072.0000 max=132385 max_rank=4 min=126642 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=61546 max_load_reduction_pct=0.317360 moved_to_replicas=228758 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] avg=131072.0000 max=132383 max_rank=0 min=126009 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=32060 max_load_reduction_pct=0.194961 moved_to_replicas=202194 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[145773, 167201, 98483, 251327, 130440, 126623, 45500, 83229] avg=131072.0000 max=251327 max_rank=3 min=45500 min_rank=6 imbalance_ratio=1.917473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] avg=131072.0000 max=132384 max_rank=3 min=127383 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=118943 max_load_reduction_pct=0.473260 moved_to_replicas=334983 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[145773, 167201, 98483, 251327, 130440, 126623, 45500, 83229] avg=131072.0000 max=251327 max_rank=3 min=45500 min_rank=6 imbalance_ratio=1.917473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[162955, 140620, 164443, 163397, 98206, 131433, 83738, 103784] avg=131072.0000 max=164443 max_rank=2 min=83738 min_rank=6 imbalance_ratio=1.254601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[162955, 140620, 164443, 163397, 98206, 131433, 83738, 103784] avg=131072.0000 max=164443 max_rank=2 min=83738 min_rank=6 imbalance_ratio=1.254601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] avg=131072.0000 max=132384 max_rank=3 min=127383 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=118943 max_load_reduction_pct=0.473260 moved_to_replicas=334983 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[162955, 140620, 164443, 163397, 98206, 131433, 83738, 103784] avg=131072.0000 max=164443 max_rank=2 min=83738 min_rank=6 imbalance_ratio=1.254601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[109726, 98274, 133489, 98157, 131060, 131064, 155513, 191293] avg=131072.0000 max=191293 max_rank=7 min=98157 min_rank=3 imbalance_ratio=1.459450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] avg=131072.0000 max=132384 max_rank=3 min=127383 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=118943 max_load_reduction_pct=0.473260 moved_to_replicas=334983 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] avg=131072.0000 max=132383 max_rank=0 min=126009 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=32060 max_load_reduction_pct=0.194961 moved_to_replicas=202194 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] avg=131072.0000 max=132383 max_rank=0 min=126009 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=32060 max_load_reduction_pct=0.194961 moved_to_replicas=202194 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[145773, 167201, 98483, 251327, 130440, 126623, 45500, 83229] avg=131072.0000 max=251327 max_rank=3 min=45500 min_rank=6 imbalance_ratio=1.917473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[162955, 140620, 164443, 163397, 98206, 131433, 83738, 103784] avg=131072.0000 max=164443 max_rank=2 min=83738 min_rank=6 imbalance_ratio=1.254601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] avg=131072.0000 max=132383 max_rank=0 min=126009 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=32060 max_load_reduction_pct=0.194961 moved_to_replicas=202194 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126002 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=58910 max_load_reduction_pct=0.307957 moved_to_replicas=161658 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[162955, 140620, 164443, 163397, 98206, 131433, 83738, 103784] avg=131072.0000 max=164443 max_rank=2 min=83738 min_rank=6 imbalance_ratio=1.254601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] avg=131072.0000 max=132383 max_rank=0 min=126009 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=32060 max_load_reduction_pct=0.194961 moved_to_replicas=202194 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[132382, 132383, 132383, 132384, 130440, 128838, 132383, 127383] avg=131072.0000 max=132384 max_rank=3 min=127383 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=118943 max_load_reduction_pct=0.473260 moved_to_replicas=334983 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[162955, 140620, 164443, 163397, 98206, 131433, 83738, 103784] avg=131072.0000 max=164443 max_rank=2 min=83738 min_rank=6 imbalance_ratio=1.254601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[109726, 98274, 133489, 98157, 131060, 131064, 155513, 191293] avg=131072.0000 max=191293 max_rank=7 min=98157 min_rank=3 imbalance_ratio=1.459450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[109726, 98274, 133489, 98157, 131060, 131064, 155513, 191293] avg=131072.0000 max=191293 max_rank=7 min=98157 min_rank=3 imbalance_ratio=1.459450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] avg=131072.0000 max=132383 max_rank=0 min=126009 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=32060 max_load_reduction_pct=0.194961 moved_to_replicas=202194 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[109726, 98274, 133489, 98157, 131060, 131064, 155513, 191293] avg=131072.0000 max=191293 max_rank=7 min=98157 min_rank=3 imbalance_ratio=1.459450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[163601, 86815, 163734, 131371, 131073, 239124, 99988, 32870] avg=131072.0000 max=239124 max_rank=5 min=32870 min_rank=7 imbalance_ratio=1.824371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] avg=131072.0000 max=132383 max_rank=0 min=126009 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=32060 max_load_reduction_pct=0.194961 moved_to_replicas=202194 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126002 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=58910 max_load_reduction_pct=0.307957 moved_to_replicas=161658 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126002 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=58910 max_load_reduction_pct=0.307957 moved_to_replicas=161658 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[109726, 98274, 133489, 98157, 131060, 131064, 155513, 191293] avg=131072.0000 max=191293 max_rank=7 min=98157 min_rank=3 imbalance_ratio=1.459450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[162955, 140620, 164443, 163397, 98206, 131433, 83738, 103784] avg=131072.0000 max=164443 max_rank=2 min=83738 min_rank=6 imbalance_ratio=1.254601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126002 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=58910 max_load_reduction_pct=0.307957 moved_to_replicas=161658 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] avg=131072.0000 max=132383 max_rank=0 min=125263 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=106741 max_load_reduction_pct=0.446383 moved_to_replicas=336392 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[109726, 98274, 133489, 98157, 131060, 131064, 155513, 191293] avg=131072.0000 max=191293 max_rank=7 min=98157 min_rank=3 imbalance_ratio=1.459450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126002 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=58910 max_load_reduction_pct=0.307957 moved_to_replicas=161658 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[109726, 98274, 133489, 98157, 131060, 131064, 155513, 191293] avg=131072.0000 max=191293 max_rank=7 min=98157 min_rank=3 imbalance_ratio=1.459450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[132383, 132382, 132383, 132383, 129220, 131433, 126009, 132383] avg=131072.0000 max=132383 max_rank=0 min=126009 min_rank=6 imbalance_ratio=1.010002 max_load_reduction=32060 max_load_reduction_pct=0.194961 moved_to_replicas=202194 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[163601, 86815, 163734, 131371, 131073, 239124, 99988, 32870] avg=131072.0000 max=239124 max_rank=5 min=32870 min_rank=7 imbalance_ratio=1.824371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[163601, 86815, 163734, 131371, 131073, 239124, 99988, 32870] avg=131072.0000 max=239124 max_rank=5 min=32870 min_rank=7 imbalance_ratio=1.824371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126002 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=58910 max_load_reduction_pct=0.307957 moved_to_replicas=161658 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[163601, 86815, 163734, 131371, 131073, 239124, 99988, 32870] avg=131072.0000 max=239124 max_rank=5 min=32870 min_rank=7 imbalance_ratio=1.824371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[57542, 98373, 163403, 164265, 131358, 205057, 163080, 65498] avg=131072.0000 max=205057 max_rank=5 min=57542 min_rank=0 imbalance_ratio=1.564461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126002 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=58910 max_load_reduction_pct=0.307957 moved_to_replicas=161658 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] avg=131072.0000 max=132383 max_rank=0 min=125263 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=106741 max_load_reduction_pct=0.446383 moved_to_replicas=336392 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] avg=131072.0000 max=132383 max_rank=0 min=125263 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=106741 max_load_reduction_pct=0.446383 moved_to_replicas=336392 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[163601, 86815, 163734, 131371, 131073, 239124, 99988, 32870] avg=131072.0000 max=239124 max_rank=5 min=32870 min_rank=7 imbalance_ratio=1.824371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] avg=131072.0000 max=132383 max_rank=0 min=125263 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=106741 max_load_reduction_pct=0.446383 moved_to_replicas=336392 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[109726, 98274, 133489, 98157, 131060, 131064, 155513, 191293] avg=131072.0000 max=191293 max_rank=7 min=98157 min_rank=3 imbalance_ratio=1.459450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] avg=131072.0000 max=132384 max_rank=5 min=129070 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72673 max_load_reduction_pct=0.354404 moved_to_replicas=334032 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[163601, 86815, 163734, 131371, 131073, 239124, 99988, 32870] avg=131072.0000 max=239124 max_rank=5 min=32870 min_rank=7 imbalance_ratio=1.824371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] avg=131072.0000 max=132383 max_rank=0 min=125263 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=106741 max_load_reduction_pct=0.446383 moved_to_replicas=336392 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[163601, 86815, 163734, 131371, 131073, 239124, 99988, 32870] avg=131072.0000 max=239124 max_rank=5 min=32870 min_rank=7 imbalance_ratio=1.824371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[57542, 98373, 163403, 164265, 131358, 205057, 163080, 65498] avg=131072.0000 max=205057 max_rank=5 min=57542 min_rank=0 imbalance_ratio=1.564461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[57542, 98373, 163403, 164265, 131358, 205057, 163080, 65498] avg=131072.0000 max=205057 max_rank=5 min=57542 min_rank=0 imbalance_ratio=1.564461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[132383, 126002, 132383, 130918, 131060, 131064, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=126002 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=58910 max_load_reduction_pct=0.307957 moved_to_replicas=161658 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[57542, 98373, 163403, 164265, 131358, 205057, 163080, 65498] avg=131072.0000 max=205057 max_rank=5 min=57542 min_rank=0 imbalance_ratio=1.564461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] avg=131072.0000 max=132383 max_rank=0 min=125263 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=106741 max_load_reduction_pct=0.446383 moved_to_replicas=336392 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[129901, 121814, 199986, 98313, 98164, 131253, 67177, 201968] avg=131072.0000 max=201968 max_rank=7 min=67177 min_rank=6 imbalance_ratio=1.540894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] avg=131072.0000 max=132383 max_rank=0 min=125263 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=106741 max_load_reduction_pct=0.446383 moved_to_replicas=336392 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] avg=131072.0000 max=132384 max_rank=5 min=129070 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72673 max_load_reduction_pct=0.354404 moved_to_replicas=334032 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[57542, 98373, 163403, 164265, 131358, 205057, 163080, 65498] avg=131072.0000 max=205057 max_rank=5 min=57542 min_rank=0 imbalance_ratio=1.564461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] avg=131072.0000 max=132384 max_rank=5 min=129070 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72673 max_load_reduction_pct=0.354404 moved_to_replicas=334032 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] avg=131072.0000 max=132384 max_rank=5 min=129070 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72673 max_load_reduction_pct=0.354404 moved_to_replicas=334032 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126968 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=69585 max_load_reduction_pct=0.344535 moved_to_replicas=277214 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[163601, 86815, 163734, 131371, 131073, 239124, 99988, 32870] avg=131072.0000 max=239124 max_rank=5 min=32870 min_rank=7 imbalance_ratio=1.824371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[57542, 98373, 163403, 164265, 131358, 205057, 163080, 65498] avg=131072.0000 max=205057 max_rank=5 min=57542 min_rank=0 imbalance_ratio=1.564461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] avg=131072.0000 max=132384 max_rank=5 min=129070 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72673 max_load_reduction_pct=0.354404 moved_to_replicas=334032 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[57542, 98373, 163403, 164265, 131358, 205057, 163080, 65498] avg=131072.0000 max=205057 max_rank=5 min=57542 min_rank=0 imbalance_ratio=1.564461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[129901, 121814, 199986, 98313, 98164, 131253, 67177, 201968] avg=131072.0000 max=201968 max_rank=7 min=67177 min_rank=6 imbalance_ratio=1.540894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[129901, 121814, 199986, 98313, 98164, 131253, 67177, 201968] avg=131072.0000 max=201968 max_rank=7 min=67177 min_rank=6 imbalance_ratio=1.540894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[129901, 121814, 199986, 98313, 98164, 131253, 67177, 201968] avg=131072.0000 max=201968 max_rank=7 min=67177 min_rank=6 imbalance_ratio=1.540894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[132383, 125263, 132383, 131371, 131073, 132383, 131339, 132381] avg=131072.0000 max=132383 max_rank=0 min=125263 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=106741 max_load_reduction_pct=0.446383 moved_to_replicas=336392 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] avg=131072.0000 max=132384 max_rank=5 min=129070 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72673 max_load_reduction_pct=0.354404 moved_to_replicas=334032 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[98328, 130965, 105392, 131715, 122175, 98119, 195601, 166281] avg=131072.0000 max=195601 max_rank=6 min=98119 min_rank=5 imbalance_ratio=1.492317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126968 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=69585 max_load_reduction_pct=0.344535 moved_to_replicas=277214 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126968 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=69585 max_load_reduction_pct=0.344535 moved_to_replicas=277214 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] avg=131072.0000 max=132384 max_rank=5 min=129070 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72673 max_load_reduction_pct=0.354404 moved_to_replicas=334032 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[129901, 121814, 199986, 98313, 98164, 131253, 67177, 201968] avg=131072.0000 max=201968 max_rank=7 min=67177 min_rank=6 imbalance_ratio=1.540894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126968 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=69585 max_load_reduction_pct=0.344535 moved_to_replicas=277214 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=128785 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=63218 max_load_reduction_pct=0.323199 moved_to_replicas=193914 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[57542, 98373, 163403, 164265, 131358, 205057, 163080, 65498] avg=131072.0000 max=205057 max_rank=5 min=57542 min_rank=0 imbalance_ratio=1.564461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[129901, 121814, 199986, 98313, 98164, 131253, 67177, 201968] avg=131072.0000 max=201968 max_rank=7 min=67177 min_rank=6 imbalance_ratio=1.540894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126968 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=69585 max_load_reduction_pct=0.344535 moved_to_replicas=277214 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[98328, 130965, 105392, 131715, 122175, 98119, 195601, 166281] avg=131072.0000 max=195601 max_rank=6 min=98119 min_rank=5 imbalance_ratio=1.492317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[98328, 130965, 105392, 131715, 122175, 98119, 195601, 166281] avg=131072.0000 max=195601 max_rank=6 min=98119 min_rank=5 imbalance_ratio=1.492317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[129901, 121814, 199986, 98313, 98164, 131253, 67177, 201968] avg=131072.0000 max=201968 max_rank=7 min=67177 min_rank=6 imbalance_ratio=1.540894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[98328, 130965, 105392, 131715, 122175, 98119, 195601, 166281] avg=131072.0000 max=195601 max_rank=6 min=98119 min_rank=5 imbalance_ratio=1.492317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[131376, 119072, 160795, 98296, 38797, 163730, 130768, 205742] avg=131072.0000 max=205742 max_rank=7 min=38797 min_rank=4 imbalance_ratio=1.569687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126968 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=69585 max_load_reduction_pct=0.344535 moved_to_replicas=277214 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[129338, 129070, 132383, 132383, 131358, 132384, 132383, 129277] avg=131072.0000 max=132384 max_rank=5 min=129070 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=72673 max_load_reduction_pct=0.354404 moved_to_replicas=334032 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=128785 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=63218 max_load_reduction_pct=0.323199 moved_to_replicas=193914 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=128785 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=63218 max_load_reduction_pct=0.323199 moved_to_replicas=193914 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126968 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=69585 max_load_reduction_pct=0.344535 moved_to_replicas=277214 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=128785 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=63218 max_load_reduction_pct=0.323199 moved_to_replicas=193914 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] avg=131072.0000 max=132385 max_rank=2 min=124517 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=73357 max_load_reduction_pct=0.356548 moved_to_replicas=262838 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[98328, 130965, 105392, 131715, 122175, 98119, 195601, 166281] avg=131072.0000 max=195601 max_rank=6 min=98119 min_rank=5 imbalance_ratio=1.492317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[98328, 130965, 105392, 131715, 122175, 98119, 195601, 166281] avg=131072.0000 max=195601 max_rank=6 min=98119 min_rank=5 imbalance_ratio=1.492317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[129901, 121814, 199986, 98313, 98164, 131253, 67177, 201968] avg=131072.0000 max=201968 max_rank=7 min=67177 min_rank=6 imbalance_ratio=1.540894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[131376, 119072, 160795, 98296, 38797, 163730, 130768, 205742] avg=131072.0000 max=205742 max_rank=7 min=38797 min_rank=4 imbalance_ratio=1.569687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[131376, 119072, 160795, 98296, 38797, 163730, 130768, 205742] avg=131072.0000 max=205742 max_rank=7 min=38797 min_rank=4 imbalance_ratio=1.569687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=128785 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=63218 max_load_reduction_pct=0.323199 moved_to_replicas=193914 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[98328, 130965, 105392, 131715, 122175, 98119, 195601, 166281] avg=131072.0000 max=195601 max_rank=6 min=98119 min_rank=5 imbalance_ratio=1.492317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[131376, 119072, 160795, 98296, 38797, 163730, 130768, 205742] avg=131072.0000 max=205742 max_rank=7 min=38797 min_rank=4 imbalance_ratio=1.569687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[65150, 228725, 49870, 195759, 73588, 217343, 53522, 164619] avg=131072.0000 max=228725 max_rank=1 min=49870 min_rank=2 imbalance_ratio=1.745033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=128785 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=63218 max_load_reduction_pct=0.323199 moved_to_replicas=193914 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] avg=131072.0000 max=132385 max_rank=2 min=124517 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=73357 max_load_reduction_pct=0.356548 moved_to_replicas=262838 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[129901, 126968, 132383, 132383, 130922, 131253, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=126968 min_rank=1 imbalance_ratio=1.010002 max_load_reduction=69585 max_load_reduction_pct=0.344535 moved_to_replicas=277214 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] avg=131072.0000 max=132385 max_rank=2 min=124517 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=73357 max_load_reduction_pct=0.356548 moved_to_replicas=262838 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=128785 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=63218 max_load_reduction_pct=0.323199 moved_to_replicas=193914 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] avg=131072.0000 max=132385 max_rank=2 min=124517 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=73357 max_load_reduction_pct=0.356548 moved_to_replicas=262838 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=125548 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96340 max_load_reduction_pct=0.421205 moved_to_replicas=557450 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[131376, 119072, 160795, 98296, 38797, 163730, 130768, 205742] avg=131072.0000 max=205742 max_rank=7 min=38797 min_rank=4 imbalance_ratio=1.569687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[131376, 119072, 160795, 98296, 38797, 163730, 130768, 205742] avg=131072.0000 max=205742 max_rank=7 min=38797 min_rank=4 imbalance_ratio=1.569687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[65150, 228725, 49870, 195759, 73588, 217343, 53522, 164619] avg=131072.0000 max=228725 max_rank=1 min=49870 min_rank=2 imbalance_ratio=1.745033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] avg=131072.0000 max=132385 max_rank=2 min=124517 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=73357 max_load_reduction_pct=0.356548 moved_to_replicas=262838 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[98328, 130965, 105392, 131715, 122175, 98119, 195601, 166281] avg=131072.0000 max=195601 max_rank=6 min=98119 min_rank=5 imbalance_ratio=1.492317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[65150, 228725, 49870, 195759, 73588, 217343, 53522, 164619] avg=131072.0000 max=228725 max_rank=1 min=49870 min_rank=2 imbalance_ratio=1.745033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[65150, 228725, 49870, 195759, 73588, 217343, 53522, 164619] avg=131072.0000 max=228725 max_rank=1 min=49870 min_rank=2 imbalance_ratio=1.745033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[131376, 119072, 160795, 98296, 38797, 163730, 130768, 205742] avg=131072.0000 max=205742 max_rank=7 min=38797 min_rank=4 imbalance_ratio=1.569687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[164437, 169647, 63096, 163662, 113079, 163886, 134126, 76643] avg=131072.0000 max=169647 max_rank=1 min=63096 min_rank=2 imbalance_ratio=1.294304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] avg=131072.0000 max=132385 max_rank=2 min=124517 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=73357 max_load_reduction_pct=0.356548 moved_to_replicas=262838 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=125548 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96340 max_load_reduction_pct=0.421205 moved_to_replicas=557450 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[128785, 130965, 132383, 131715, 129082, 130880, 132383, 132383] avg=131072.0000 max=132383 max_rank=2 min=128785 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=63218 max_load_reduction_pct=0.323199 moved_to_replicas=193914 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=125548 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96340 max_load_reduction_pct=0.421205 moved_to_replicas=557450 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=125548 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96340 max_load_reduction_pct=0.421205 moved_to_replicas=557450 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] avg=131072.0000 max=132385 max_rank=2 min=124517 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=73357 max_load_reduction_pct=0.356548 moved_to_replicas=262838 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] avg=131072.0000 max=132383 max_rank=0 min=126926 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=37264 max_load_reduction_pct=0.219656 moved_to_replicas=269723 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[65150, 228725, 49870, 195759, 73588, 217343, 53522, 164619] avg=131072.0000 max=228725 max_rank=1 min=49870 min_rank=2 imbalance_ratio=1.745033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[65150, 228725, 49870, 195759, 73588, 217343, 53522, 164619] avg=131072.0000 max=228725 max_rank=1 min=49870 min_rank=2 imbalance_ratio=1.745033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[164437, 169647, 63096, 163662, 113079, 163886, 134126, 76643] avg=131072.0000 max=169647 max_rank=1 min=63096 min_rank=2 imbalance_ratio=1.294304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=125548 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96340 max_load_reduction_pct=0.421205 moved_to_replicas=557450 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[164437, 169647, 63096, 163662, 113079, 163886, 134126, 76643] avg=131072.0000 max=169647 max_rank=1 min=63096 min_rank=2 imbalance_ratio=1.294304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[131376, 119072, 160795, 98296, 38797, 163730, 130768, 205742] avg=131072.0000 max=205742 max_rank=7 min=38797 min_rank=4 imbalance_ratio=1.569687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[164437, 169647, 63096, 163662, 113079, 163886, 134126, 76643] avg=131072.0000 max=169647 max_rank=1 min=63096 min_rank=2 imbalance_ratio=1.294304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[65150, 228725, 49870, 195759, 73588, 217343, 53522, 164619] avg=131072.0000 max=228725 max_rank=1 min=49870 min_rank=2 imbalance_ratio=1.745033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[138721, 196556, 65635, 163622, 130624, 92117, 131209, 130092] avg=131072.0000 max=196556 max_rank=1 min=65635 min_rank=2 imbalance_ratio=1.499603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=125548 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96340 max_load_reduction_pct=0.421205 moved_to_replicas=557450 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] avg=131072.0000 max=132383 max_rank=0 min=126926 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=37264 max_load_reduction_pct=0.219656 moved_to_replicas=269723 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] avg=131072.0000 max=132383 max_rank=0 min=126926 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=37264 max_load_reduction_pct=0.219656 moved_to_replicas=269723 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[131376, 124517, 132385, 132382, 132383, 132383, 130768, 132382] avg=131072.0000 max=132385 max_rank=2 min=124517 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=73357 max_load_reduction_pct=0.356548 moved_to_replicas=262838 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] avg=131072.0000 max=132383 max_rank=0 min=126926 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=37264 max_load_reduction_pct=0.219656 moved_to_replicas=269723 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=125548 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96340 max_load_reduction_pct=0.421205 moved_to_replicas=557450 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] avg=131072.0000 max=132383 max_rank=1 min=129635 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64173 max_load_reduction_pct=0.326487 moved_to_replicas=205335 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[164437, 169647, 63096, 163662, 113079, 163886, 134126, 76643] avg=131072.0000 max=169647 max_rank=1 min=63096 min_rank=2 imbalance_ratio=1.294304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[138721, 196556, 65635, 163622, 130624, 92117, 131209, 130092] avg=131072.0000 max=196556 max_rank=1 min=65635 min_rank=2 imbalance_ratio=1.499603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[164437, 169647, 63096, 163662, 113079, 163886, 134126, 76643] avg=131072.0000 max=169647 max_rank=1 min=63096 min_rank=2 imbalance_ratio=1.294304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] avg=131072.0000 max=132383 max_rank=0 min=126926 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=37264 max_load_reduction_pct=0.219656 moved_to_replicas=269723 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[138721, 196556, 65635, 163622, 130624, 92117, 131209, 130092] avg=131072.0000 max=196556 max_rank=1 min=65635 min_rank=2 imbalance_ratio=1.499603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[138721, 196556, 65635, 163622, 130624, 92117, 131209, 130092] avg=131072.0000 max=196556 max_rank=1 min=65635 min_rank=2 imbalance_ratio=1.499603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[164437, 169647, 63096, 163662, 113079, 163886, 134126, 76643] avg=131072.0000 max=169647 max_rank=1 min=63096 min_rank=2 imbalance_ratio=1.294304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[65150, 228725, 49870, 195759, 73588, 217343, 53522, 164619] avg=131072.0000 max=228725 max_rank=1 min=49870 min_rank=2 imbalance_ratio=1.745033 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[65658, 163780, 97991, 163771, 98288, 196892, 195332, 66864] avg=131072.0000 max=196892 max_rank=5 min=65658 min_rank=0 imbalance_ratio=1.502167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] avg=131072.0000 max=132383 max_rank=1 min=129635 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64173 max_load_reduction_pct=0.326487 moved_to_replicas=205335 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] avg=131072.0000 max=132383 max_rank=0 min=126926 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=37264 max_load_reduction_pct=0.219656 moved_to_replicas=269723 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] avg=131072.0000 max=132383 max_rank=1 min=129635 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64173 max_load_reduction_pct=0.326487 moved_to_replicas=205335 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] avg=131072.0000 max=132383 max_rank=1 min=129635 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64173 max_load_reduction_pct=0.326487 moved_to_replicas=205335 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] avg=131072.0000 max=132383 max_rank=0 min=126926 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=37264 max_load_reduction_pct=0.219656 moved_to_replicas=269723 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[128731, 132383, 125548, 132383, 132382, 132381, 132385, 132383] avg=131072.0000 max=132385 max_rank=6 min=125548 min_rank=2 imbalance_ratio=1.010017 max_load_reduction=96340 max_load_reduction_pct=0.421205 moved_to_replicas=557450 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] avg=131072.0000 max=132383 max_rank=1 min=128476 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64509 max_load_reduction_pct=0.327636 moved_to_replicas=381580 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[138721, 196556, 65635, 163622, 130624, 92117, 131209, 130092] avg=131072.0000 max=196556 max_rank=1 min=65635 min_rank=2 imbalance_ratio=1.499603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[65658, 163780, 97991, 163771, 98288, 196892, 195332, 66864] avg=131072.0000 max=196892 max_rank=5 min=65658 min_rank=0 imbalance_ratio=1.502167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[138721, 196556, 65635, 163622, 130624, 92117, 131209, 130092] avg=131072.0000 max=196556 max_rank=1 min=65635 min_rank=2 imbalance_ratio=1.499603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] avg=131072.0000 max=132383 max_rank=1 min=129635 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64173 max_load_reduction_pct=0.326487 moved_to_replicas=205335 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[65658, 163780, 97991, 163771, 98288, 196892, 195332, 66864] avg=131072.0000 max=196892 max_rank=5 min=65658 min_rank=0 imbalance_ratio=1.502167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[65658, 163780, 97991, 163771, 98288, 196892, 195332, 66864] avg=131072.0000 max=196892 max_rank=5 min=65658 min_rank=0 imbalance_ratio=1.502167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[138721, 196556, 65635, 163622, 130624, 92117, 131209, 130092] avg=131072.0000 max=196556 max_rank=1 min=65635 min_rank=2 imbalance_ratio=1.499603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[131062, 98425, 111663, 67205, 131133, 214188, 229176, 65724] avg=131072.0000 max=229176 max_rank=6 min=65724 min_rank=7 imbalance_ratio=1.748474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[164437, 169647, 63096, 163662, 113079, 163886, 134126, 76643] avg=131072.0000 max=169647 max_rank=1 min=63096 min_rank=2 imbalance_ratio=1.294304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] avg=131072.0000 max=132383 max_rank=1 min=128476 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64509 max_load_reduction_pct=0.327636 moved_to_replicas=381580 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] avg=131072.0000 max=132383 max_rank=1 min=129635 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64173 max_load_reduction_pct=0.326487 moved_to_replicas=205335 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] avg=131072.0000 max=132383 max_rank=1 min=128476 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64509 max_load_reduction_pct=0.327636 moved_to_replicas=381580 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] avg=131072.0000 max=132383 max_rank=1 min=128476 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64509 max_load_reduction_pct=0.327636 moved_to_replicas=381580 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] avg=131072.0000 max=132383 max_rank=1 min=129635 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64173 max_load_reduction_pct=0.326487 moved_to_replicas=205335 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] avg=131072.0000 max=132383 max_rank=3 min=128289 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96793 max_load_reduction_pct=0.422352 moved_to_replicas=360317 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[65658, 163780, 97991, 163771, 98288, 196892, 195332, 66864] avg=131072.0000 max=196892 max_rank=5 min=65658 min_rank=0 imbalance_ratio=1.502167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[132383, 132382, 127354, 132383, 132382, 132383, 132383, 126926] avg=131072.0000 max=132383 max_rank=0 min=126926 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=37264 max_load_reduction_pct=0.219656 moved_to_replicas=269723 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[131062, 98425, 111663, 67205, 131133, 214188, 229176, 65724] avg=131072.0000 max=229176 max_rank=6 min=65724 min_rank=7 imbalance_ratio=1.748474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[65658, 163780, 97991, 163771, 98288, 196892, 195332, 66864] avg=131072.0000 max=196892 max_rank=5 min=65658 min_rank=0 imbalance_ratio=1.502167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[131062, 98425, 111663, 67205, 131133, 214188, 229176, 65724] avg=131072.0000 max=229176 max_rank=6 min=65724 min_rank=7 imbalance_ratio=1.748474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] avg=131072.0000 max=132383 max_rank=1 min=128476 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64509 max_load_reduction_pct=0.327636 moved_to_replicas=381580 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[131062, 98425, 111663, 67205, 131133, 214188, 229176, 65724] avg=131072.0000 max=229176 max_rank=6 min=65724 min_rank=7 imbalance_ratio=1.748474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[65658, 163780, 97991, 163771, 98288, 196892, 195332, 66864] avg=131072.0000 max=196892 max_rank=5 min=65658 min_rank=0 imbalance_ratio=1.502167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[75652, 196586, 65664, 132947, 50419, 198671, 131253, 197384] avg=131072.0000 max=198671 max_rank=5 min=50419 min_rank=4 imbalance_ratio=1.515739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] avg=131072.0000 max=132383 max_rank=3 min=128289 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96793 max_load_reduction_pct=0.422352 moved_to_replicas=360317 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[138721, 196556, 65635, 163622, 130624, 92117, 131209, 130092] avg=131072.0000 max=196556 max_rank=1 min=65635 min_rank=2 imbalance_ratio=1.499603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] avg=131072.0000 max=132383 max_rank=1 min=128476 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64509 max_load_reduction_pct=0.327636 moved_to_replicas=381580 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] avg=131072.0000 max=132383 max_rank=3 min=128289 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96793 max_load_reduction_pct=0.422352 moved_to_replicas=360317 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] avg=131072.0000 max=132383 max_rank=3 min=128289 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96793 max_load_reduction_pct=0.422352 moved_to_replicas=360317 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] avg=131072.0000 max=132383 max_rank=1 min=128476 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64509 max_load_reduction_pct=0.327636 moved_to_replicas=381580 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] avg=131072.0000 max=132383 max_rank=1 min=124247 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66288 max_load_reduction_pct=0.333657 moved_to_replicas=389984 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[132381, 132383, 129635, 132383, 130624, 129869, 131209, 130092] avg=131072.0000 max=132383 max_rank=1 min=129635 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=64173 max_load_reduction_pct=0.326487 moved_to_replicas=205335 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[131062, 98425, 111663, 67205, 131133, 214188, 229176, 65724] avg=131072.0000 max=229176 max_rank=6 min=65724 min_rank=7 imbalance_ratio=1.748474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[75652, 196586, 65664, 132947, 50419, 198671, 131253, 197384] avg=131072.0000 max=198671 max_rank=5 min=50419 min_rank=4 imbalance_ratio=1.515739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[131062, 98425, 111663, 67205, 131133, 214188, 229176, 65724] avg=131072.0000 max=229176 max_rank=6 min=65724 min_rank=7 imbalance_ratio=1.748474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[75652, 196586, 65664, 132947, 50419, 198671, 131253, 197384] avg=131072.0000 max=198671 max_rank=5 min=50419 min_rank=4 imbalance_ratio=1.515739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[75652, 196586, 65664, 132947, 50419, 198671, 131253, 197384] avg=131072.0000 max=198671 max_rank=5 min=50419 min_rank=4 imbalance_ratio=1.515739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] avg=131072.0000 max=132383 max_rank=3 min=128289 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96793 max_load_reduction_pct=0.422352 moved_to_replicas=360317 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[131062, 98425, 111663, 67205, 131133, 214188, 229176, 65724] avg=131072.0000 max=229176 max_rank=6 min=65724 min_rank=7 imbalance_ratio=1.748474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] avg=131072.0000 max=132383 max_rank=1 min=124247 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66288 max_load_reduction_pct=0.333657 moved_to_replicas=389984 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] avg=131072.0000 max=132383 max_rank=3 min=128289 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96793 max_load_reduction_pct=0.422352 moved_to_replicas=360317 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[65658, 163780, 97991, 163771, 98288, 196892, 195332, 66864] avg=131072.0000 max=196892 max_rank=5 min=65658 min_rank=0 imbalance_ratio=1.502167 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] avg=131072.0000 max=132383 max_rank=1 min=124247 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66288 max_load_reduction_pct=0.333657 moved_to_replicas=389984 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] avg=131072.0000 max=132383 max_rank=1 min=124247 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66288 max_load_reduction_pct=0.333657 moved_to_replicas=389984 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] avg=131072.0000 max=132383 max_rank=3 min=128289 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96793 max_load_reduction_pct=0.422352 moved_to_replicas=360317 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[129815, 132383, 130752, 132383, 128476, 132383, 132383, 130001] avg=131072.0000 max=132383 max_rank=1 min=128476 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=64509 max_load_reduction_pct=0.327636 moved_to_replicas=381580 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[75652, 196586, 65664, 132947, 50419, 198671, 131253, 197384] avg=131072.0000 max=198671 max_rank=5 min=50419 min_rank=4 imbalance_ratio=1.515739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[75652, 196586, 65664, 132947, 50419, 198671, 131253, 197384] avg=131072.0000 max=198671 max_rank=5 min=50419 min_rank=4 imbalance_ratio=1.515739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] avg=131072.0000 max=132383 max_rank=1 min=124247 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66288 max_load_reduction_pct=0.333657 moved_to_replicas=389984 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[75652, 196586, 65664, 132947, 50419, 198671, 131253, 197384] avg=131072.0000 max=198671 max_rank=5 min=50419 min_rank=4 imbalance_ratio=1.515739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] avg=131072.0000 max=132383 max_rank=1 min=124247 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66288 max_load_reduction_pct=0.333657 moved_to_replicas=389984 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[131062, 98425, 111663, 67205, 131133, 214188, 229176, 65724] avg=131072.0000 max=229176 max_rank=6 min=65724 min_rank=7 imbalance_ratio=1.748474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] avg=131072.0000 max=132383 max_rank=1 min=124247 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66288 max_load_reduction_pct=0.333657 moved_to_replicas=389984 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[131062, 129699, 128289, 132383, 131133, 132383, 132383, 131244] avg=131072.0000 max=132383 max_rank=3 min=128289 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96793 max_load_reduction_pct=0.422352 moved_to_replicas=360317 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[75652, 196586, 65664, 132947, 50419, 198671, 131253, 197384] avg=131072.0000 max=198671 max_rank=5 min=50419 min_rank=4 imbalance_ratio=1.515739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[132381, 132383, 131163, 132383, 124247, 132383, 131253, 132383] avg=131072.0000 max=132383 max_rank=1 min=124247 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=66288 max_load_reduction_pct=0.333657 moved_to_replicas=389984 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=4221998 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=996736 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=4189908 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1023872 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=4193640 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1023616 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=4197068 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018368 peak_demand=132386 required_capacity=1.010025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=4211718 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1007104 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=4169539 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1049984 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=4195893 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1021056 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=4174668 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1042304 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:17:56] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 576.1 | throughput per GPU (TFLOP/s/GPU): 578.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.919160E+00 | loss scale: 1.0 | grad norm: 5.200 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262661 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=163840 expected_recv=252953 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=163840 expected_recv=259284 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=163840 expected_recv=263191 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262661 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=163840 expected_recv=252953 actual_valid_recv=254976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=254976 slack_slots=72704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=163840 expected_recv=259284 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=163840 expected_recv=263191 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=163840 expected_recv=263683 expected_overflow=0 exact_recv_per_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=163840 expected_recv=258669 expected_overflow=0 exact_recv_per_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=163840 expected_recv=263683 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=163840 expected_recv=260037 expected_overflow=0 exact_recv_per_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=163840 expected_recv=258669 actual_valid_recv=261248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=163840 expected_recv=255695 expected_overflow=0 exact_recv_per_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=163840 expected_recv=260037 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=261248 slack_slots=66432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=163840 expected_recv=255695 actual_valid_recv=258560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=258560 slack_slots=69120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=163840 expected_recv=263686 expected_overflow=0 exact_recv_per_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=163840 expected_recv=258174 expected_overflow=0 exact_recv_per_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=163840 expected_recv=263686 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=163840 expected_recv=258174 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=260736 slack_slots=66944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=163840 expected_recv=259739 expected_overflow=0 exact_recv_per_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=163840 expected_recv=260689 expected_overflow=0 exact_recv_per_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=163840 expected_recv=260567 expected_overflow=0 exact_recv_per_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=163840 expected_recv=259739 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=163840 expected_recv=260689 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=163840 expected_recv=260567 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=163840 expected_recv=261735 expected_overflow=0 exact_recv_per_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=163840 expected_recv=261735 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=163840 expected_recv=258625 expected_overflow=0 exact_recv_per_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=163840 expected_recv=252957 expected_overflow=0 exact_recv_per_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=163840 expected_recv=258625 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=163840 expected_recv=250012 expected_overflow=0 exact_recv_per_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=163840 expected_recv=252957 actual_valid_recv=255872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=163840 expected_recv=264003 expected_overflow=0 exact_recv_per_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=163840 expected_recv=250012 actual_valid_recv=252416 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=255872 slack_slots=71808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=163840 expected_recv=264003 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=252416 slack_slots=75264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262045 expected_overflow=0 exact_recv_per_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262045 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262029 expected_overflow=0 exact_recv_per_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262029 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264588 expected_overflow=0 exact_recv_per_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=163840 expected_recv=247786 expected_overflow=0 exact_recv_per_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264588 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=163840 expected_recv=247786 actual_valid_recv=249856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=163840 expected_recv=261048 expected_overflow=0 exact_recv_per_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=249856 slack_slots=77824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=163840 expected_recv=261048 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264665 expected_overflow=0 exact_recv_per_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264665 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=163840 expected_recv=264763 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=163840 expected_recv=255845 expected_overflow=0 exact_recv_per_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=163840 expected_recv=255845 actual_valid_recv=258432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=258432 slack_slots=69248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262125 expected_overflow=0 exact_recv_per_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262125 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=163840 expected_recv=260505 expected_overflow=0 exact_recv_per_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=163840 expected_recv=252709 expected_overflow=0 exact_recv_per_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=163840 expected_recv=260505 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=163840 expected_recv=260310 expected_overflow=0 exact_recv_per_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=163840 expected_recv=252709 actual_valid_recv=254720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=163840 expected_recv=260310 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=254720 slack_slots=72960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=163840 expected_recv=256943 expected_overflow=0 exact_recv_per_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=163840 expected_recv=259715 expected_overflow=0 exact_recv_per_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262970 expected_overflow=0 exact_recv_per_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=163840 expected_recv=256943 actual_valid_recv=259456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=163840 expected_recv=259715 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=163840 expected_recv=264762 expected_overflow=0 exact_recv_per_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262970 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=163840 expected_recv=264773 expected_overflow=0 exact_recv_per_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=259456 slack_slots=68224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=260864 slack_slots=66816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=163840 expected_recv=260903 expected_overflow=0 exact_recv_per_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=163840 expected_recv=264762 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265472 slack_slots=62208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=163840 expected_recv=264773 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=163840 expected_recv=260903 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=163840 expected_recv=257219 expected_overflow=0 exact_recv_per_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=268032 slack_slots=59648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=163840 expected_recv=257219 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=259584 slack_slots=68096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=163840 expected_recv=258859 expected_overflow=0 exact_recv_per_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=163840 expected_recv=260250 expected_overflow=0 exact_recv_per_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=163840 expected_recv=258859 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=163840 expected_recv=260250 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=163840 expected_recv=254747 expected_overflow=0 exact_recv_per_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=163840 expected_recv=254747 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=257408 slack_slots=70272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=163840 expected_recv=259715 expected_overflow=0 exact_recv_per_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=163840 expected_recv=254209 expected_overflow=0 exact_recv_per_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=163840 expected_recv=259715 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=163840 expected_recv=258410 expected_overflow=0 exact_recv_per_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=163840 expected_recv=254209 actual_valid_recv=256384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=261760 slack_slots=65920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=163840 expected_recv=258410 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=256384 slack_slots=71296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=163840 expected_recv=260704 expected_overflow=0 exact_recv_per_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=163840 expected_recv=260704 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=163840 expected_recv=259286 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=163840 expected_recv=259286 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=163840 expected_recv=259616 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=163840 expected_recv=259616 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=163840 expected_recv=259955 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=261760 slack_slots=65920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=163840 expected_recv=250366 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=163840 expected_recv=259955 actual_valid_recv=261760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=163840 expected_recv=259231 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=163840 expected_recv=250366 actual_valid_recv=253568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=261760 slack_slots=65920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=163840 expected_recv=259231 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=253568 slack_slots=74112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=265856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=163840 expected_recv=256743 expected_overflow=0 exact_recv_per_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265856 slack_slots=61824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264389 expected_overflow=0 exact_recv_per_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=163840 expected_recv=256743 actual_valid_recv=260096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264389 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=260096 slack_slots=67584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=163840 expected_recv=258191 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=163840 expected_recv=258191 actual_valid_recv=260224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=163840 expected_recv=264555 expected_overflow=0 exact_recv_per_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=260224 slack_slots=67456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=163840 expected_recv=264555 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264763 expected_overflow=0 exact_recv_per_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=163840 expected_recv=261638 expected_overflow=0 exact_recv_per_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264763 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=163840 expected_recv=261638 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=163840 expected_recv=256676 expected_overflow=0 exact_recv_per_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=163840 expected_recv=252191 expected_overflow=0 exact_recv_per_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=163840 expected_recv=256676 actual_valid_recv=259072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=163840 expected_recv=252191 actual_valid_recv=254848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=163840 expected_recv=262203 expected_overflow=0 exact_recv_per_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=259072 slack_slots=68608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=254848 slack_slots=72832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=163840 expected_recv=262203 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=163840 expected_recv=261446 expected_overflow=0 exact_recv_per_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=163840 expected_recv=261446 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=163840 expected_recv=258660 expected_overflow=0 exact_recv_per_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=163840 expected_recv=260354 expected_overflow=0 exact_recv_per_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=163840 expected_recv=258660 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=163840 expected_recv=263893 expected_overflow=0 exact_recv_per_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=261120 slack_slots=66560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=163840 expected_recv=260354 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=163840 expected_recv=263893 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=163840 expected_recv=258655 expected_overflow=0 exact_recv_per_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=163840 expected_recv=258655 actual_valid_recv=260992 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[153821, 139984, 125179, 95835, 139868, 109588, 130808, 153493] avg=131072.0000 max=153821 max_rank=0 min=95835 min_rank=3 imbalance_ratio=1.173561 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=260992 slack_slots=66688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[153821, 139984, 125179, 95835, 139868, 109588, 130808, 153493] avg=131072.0000 max=153821 max_rank=0 min=95835 min_rank=3 imbalance_ratio=1.173561 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=163840 expected_recv=257629 expected_overflow=0 exact_recv_per_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[153821, 139984, 125179, 95835, 139868, 109588, 130808, 153493] avg=131072.0000 max=153821 max_rank=0 min=95835 min_rank=3 imbalance_ratio=1.173561 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] avg=131072.0000 max=132383 max_rank=0 min=125179 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=21438 max_load_reduction_pct=0.139370 moved_to_replicas=117107 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=163840 expected_recv=257629 actual_valid_recv=260480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=260480 slack_slots=67200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[153821, 139984, 125179, 95835, 139868, 109588, 130808, 153493] avg=131072.0000 max=153821 max_rank=0 min=95835 min_rank=3 imbalance_ratio=1.173561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[153821, 139984, 125179, 95835, 139868, 109588, 130808, 153493] avg=131072.0000 max=153821 max_rank=0 min=95835 min_rank=3 imbalance_ratio=1.173561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] avg=131072.0000 max=132383 max_rank=0 min=125179 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=21438 max_load_reduction_pct=0.139370 moved_to_replicas=117107 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] avg=131072.0000 max=132383 max_rank=0 min=125179 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=21438 max_load_reduction_pct=0.139370 moved_to_replicas=117107 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[67084, 127621, 175699, 187820, 110614, 103892, 183310, 92536] avg=131072.0000 max=187820 max_rank=3 min=67084 min_rank=0 imbalance_ratio=1.432953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] avg=131072.0000 max=132383 max_rank=0 min=125179 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=21438 max_load_reduction_pct=0.139370 moved_to_replicas=117107 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] avg=131072.0000 max=132383 max_rank=0 min=125179 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=21438 max_load_reduction_pct=0.139370 moved_to_replicas=117107 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[153821, 139984, 125179, 95835, 139868, 109588, 130808, 153493] avg=131072.0000 max=153821 max_rank=0 min=95835 min_rank=3 imbalance_ratio=1.173561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[67084, 127621, 175699, 187820, 110614, 103892, 183310, 92536] avg=131072.0000 max=187820 max_rank=3 min=67084 min_rank=0 imbalance_ratio=1.432953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[153821, 139984, 125179, 95835, 139868, 109588, 130808, 153493] avg=131072.0000 max=153821 max_rank=0 min=95835 min_rank=3 imbalance_ratio=1.173561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] avg=131072.0000 max=132384 max_rank=2 min=126657 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=55436 max_load_reduction_pct=0.295155 moved_to_replicas=301578 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[67084, 127621, 175699, 187820, 110614, 103892, 183310, 92536] avg=131072.0000 max=187820 max_rank=3 min=67084 min_rank=0 imbalance_ratio=1.432953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] avg=131072.0000 max=132383 max_rank=0 min=125179 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=21438 max_load_reduction_pct=0.139370 moved_to_replicas=117107 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] avg=131072.0000 max=132384 max_rank=2 min=126657 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=55436 max_load_reduction_pct=0.295155 moved_to_replicas=301578 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[67084, 127621, 175699, 187820, 110614, 103892, 183310, 92536] avg=131072.0000 max=187820 max_rank=3 min=67084 min_rank=0 imbalance_ratio=1.432953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[153821, 139984, 125179, 95835, 139868, 109588, 130808, 153493] avg=131072.0000 max=153821 max_rank=0 min=95835 min_rank=3 imbalance_ratio=1.173561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[67084, 127621, 175699, 187820, 110614, 103892, 183310, 92536] avg=131072.0000 max=187820 max_rank=3 min=67084 min_rank=0 imbalance_ratio=1.432953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] avg=131072.0000 max=132383 max_rank=0 min=125179 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=21438 max_load_reduction_pct=0.139370 moved_to_replicas=117107 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] avg=131072.0000 max=132384 max_rank=2 min=126657 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=55436 max_load_reduction_pct=0.295155 moved_to_replicas=301578 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[131302, 192057, 98529, 257125, 119029, 133823, 49785, 66926] avg=131072.0000 max=257125 max_rank=3 min=49785 min_rank=6 imbalance_ratio=1.961708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] avg=131072.0000 max=132384 max_rank=2 min=126657 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=55436 max_load_reduction_pct=0.295155 moved_to_replicas=301578 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] avg=131072.0000 max=132384 max_rank=2 min=126657 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=55436 max_load_reduction_pct=0.295155 moved_to_replicas=301578 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[132383, 132383, 125179, 132381, 132383, 130676, 130808, 132383] avg=131072.0000 max=132383 max_rank=0 min=125179 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=21438 max_load_reduction_pct=0.139370 moved_to_replicas=117107 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[67084, 127621, 175699, 187820, 110614, 103892, 183310, 92536] avg=131072.0000 max=187820 max_rank=3 min=67084 min_rank=0 imbalance_ratio=1.432953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[131302, 192057, 98529, 257125, 119029, 133823, 49785, 66926] avg=131072.0000 max=257125 max_rank=3 min=49785 min_rank=6 imbalance_ratio=1.961708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] avg=131072.0000 max=132383 max_rank=5 min=126437 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=124742 max_load_reduction_pct=0.485141 moved_to_replicas=385126 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[131302, 192057, 98529, 257125, 119029, 133823, 49785, 66926] avg=131072.0000 max=257125 max_rank=3 min=49785 min_rank=6 imbalance_ratio=1.961708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[67084, 127621, 175699, 187820, 110614, 103892, 183310, 92536] avg=131072.0000 max=187820 max_rank=3 min=67084 min_rank=0 imbalance_ratio=1.432953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] avg=131072.0000 max=132383 max_rank=5 min=126437 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=124742 max_load_reduction_pct=0.485141 moved_to_replicas=385126 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] avg=131072.0000 max=132384 max_rank=2 min=126657 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=55436 max_load_reduction_pct=0.295155 moved_to_replicas=301578 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[131302, 192057, 98529, 257125, 119029, 133823, 49785, 66926] avg=131072.0000 max=257125 max_rank=3 min=49785 min_rank=6 imbalance_ratio=1.961708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[131302, 192057, 98529, 257125, 119029, 133823, 49785, 66926] avg=131072.0000 max=257125 max_rank=3 min=49785 min_rank=6 imbalance_ratio=1.961708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[67084, 127621, 175699, 187820, 110614, 103892, 183310, 92536] avg=131072.0000 max=187820 max_rank=3 min=67084 min_rank=0 imbalance_ratio=1.432953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] avg=131072.0000 max=132383 max_rank=5 min=126437 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=124742 max_load_reduction_pct=0.485141 moved_to_replicas=385126 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] avg=131072.0000 max=132384 max_rank=2 min=126657 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=55436 max_load_reduction_pct=0.295155 moved_to_replicas=301578 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[155115, 129351, 144936, 177045, 99410, 159373, 94603, 88743] avg=131072.0000 max=177045 max_rank=3 min=88743 min_rank=7 imbalance_ratio=1.350746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] avg=131072.0000 max=132383 max_rank=5 min=126437 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=124742 max_load_reduction_pct=0.485141 moved_to_replicas=385126 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] avg=131072.0000 max=132383 max_rank=5 min=126437 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=124742 max_load_reduction_pct=0.485141 moved_to_replicas=385126 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[155115, 129351, 144936, 177045, 99410, 159373, 94603, 88743] avg=131072.0000 max=177045 max_rank=3 min=88743 min_rank=7 imbalance_ratio=1.350746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[132383, 127621, 132384, 132384, 132382, 132383, 132382, 126657] avg=131072.0000 max=132384 max_rank=2 min=126657 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=55436 max_load_reduction_pct=0.295155 moved_to_replicas=301578 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[131302, 192057, 98529, 257125, 119029, 133823, 49785, 66926] avg=131072.0000 max=257125 max_rank=3 min=49785 min_rank=6 imbalance_ratio=1.961708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] avg=131072.0000 max=132384 max_rank=5 min=128078 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=44661 max_load_reduction_pct=0.252258 moved_to_replicas=215806 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[155115, 129351, 144936, 177045, 99410, 159373, 94603, 88743] avg=131072.0000 max=177045 max_rank=3 min=88743 min_rank=7 imbalance_ratio=1.350746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[131302, 192057, 98529, 257125, 119029, 133823, 49785, 66926] avg=131072.0000 max=257125 max_rank=3 min=49785 min_rank=6 imbalance_ratio=1.961708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] avg=131072.0000 max=132384 max_rank=5 min=128078 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=44661 max_load_reduction_pct=0.252258 moved_to_replicas=215806 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[155115, 129351, 144936, 177045, 99410, 159373, 94603, 88743] avg=131072.0000 max=177045 max_rank=3 min=88743 min_rank=7 imbalance_ratio=1.350746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] avg=131072.0000 max=132383 max_rank=5 min=126437 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=124742 max_load_reduction_pct=0.485141 moved_to_replicas=385126 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[155115, 129351, 144936, 177045, 99410, 159373, 94603, 88743] avg=131072.0000 max=177045 max_rank=3 min=88743 min_rank=7 imbalance_ratio=1.350746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] avg=131072.0000 max=132384 max_rank=5 min=128078 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=44661 max_load_reduction_pct=0.252258 moved_to_replicas=215806 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[131620, 103928, 175808, 65993, 131050, 131027, 160939, 148211] avg=131072.0000 max=175808 max_rank=2 min=65993 min_rank=3 imbalance_ratio=1.341309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[131302, 192057, 98529, 257125, 119029, 133823, 49785, 66926] avg=131072.0000 max=257125 max_rank=3 min=49785 min_rank=6 imbalance_ratio=1.961708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] avg=131072.0000 max=132383 max_rank=5 min=126437 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=124742 max_load_reduction_pct=0.485141 moved_to_replicas=385126 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] avg=131072.0000 max=132384 max_rank=5 min=128078 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=44661 max_load_reduction_pct=0.252258 moved_to_replicas=215806 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] avg=131072.0000 max=132384 max_rank=5 min=128078 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=44661 max_load_reduction_pct=0.252258 moved_to_replicas=215806 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[131620, 103928, 175808, 65993, 131050, 131027, 160939, 148211] avg=131072.0000 max=175808 max_rank=2 min=65993 min_rank=3 imbalance_ratio=1.341309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125346 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=43423 max_load_reduction_pct=0.246991 moved_to_replicas=176291 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[155115, 129351, 144936, 177045, 99410, 159373, 94603, 88743] avg=131072.0000 max=177045 max_rank=3 min=88743 min_rank=7 imbalance_ratio=1.350746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[131620, 103928, 175808, 65993, 131050, 131027, 160939, 148211] avg=131072.0000 max=175808 max_rank=2 min=65993 min_rank=3 imbalance_ratio=1.341309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[131302, 132382, 126437, 132382, 128925, 132383, 132382, 132383] avg=131072.0000 max=132383 max_rank=5 min=126437 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=124742 max_load_reduction_pct=0.485141 moved_to_replicas=385126 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125346 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=43423 max_load_reduction_pct=0.246991 moved_to_replicas=176291 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[131620, 103928, 175808, 65993, 131050, 131027, 160939, 148211] avg=131072.0000 max=175808 max_rank=2 min=65993 min_rank=3 imbalance_ratio=1.341309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[155115, 129351, 144936, 177045, 99410, 159373, 94603, 88743] avg=131072.0000 max=177045 max_rank=3 min=88743 min_rank=7 imbalance_ratio=1.350746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] avg=131072.0000 max=132384 max_rank=5 min=128078 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=44661 max_load_reduction_pct=0.252258 moved_to_replicas=215806 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[103631, 97885, 123905, 158904, 160957, 250065, 120411, 32818] avg=131072.0000 max=250065 max_rank=5 min=32818 min_rank=7 imbalance_ratio=1.907845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[131620, 103928, 175808, 65993, 131050, 131027, 160939, 148211] avg=131072.0000 max=175808 max_rank=2 min=65993 min_rank=3 imbalance_ratio=1.341309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125346 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=43423 max_load_reduction_pct=0.246991 moved_to_replicas=176291 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125346 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=43423 max_load_reduction_pct=0.246991 moved_to_replicas=176291 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] avg=131072.0000 max=132384 max_rank=5 min=128078 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=44661 max_load_reduction_pct=0.252258 moved_to_replicas=215806 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[155115, 129351, 144936, 177045, 99410, 159373, 94603, 88743] avg=131072.0000 max=177045 max_rank=3 min=88743 min_rank=7 imbalance_ratio=1.350746 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[103631, 97885, 123905, 158904, 160957, 250065, 120411, 32818] avg=131072.0000 max=250065 max_rank=5 min=32818 min_rank=7 imbalance_ratio=1.907845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] avg=131072.0000 max=132384 max_rank=3 min=123905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=117681 max_load_reduction_pct=0.470602 moved_to_replicas=346542 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125346 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=43423 max_load_reduction_pct=0.246991 moved_to_replicas=176291 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[131620, 103928, 175808, 65993, 131050, 131027, 160939, 148211] avg=131072.0000 max=175808 max_rank=2 min=65993 min_rank=3 imbalance_ratio=1.341309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[103631, 97885, 123905, 158904, 160957, 250065, 120411, 32818] avg=131072.0000 max=250065 max_rank=5 min=32818 min_rank=7 imbalance_ratio=1.907845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[132383, 129351, 132383, 132383, 128078, 132384, 129231, 132383] avg=131072.0000 max=132384 max_rank=5 min=128078 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=44661 max_load_reduction_pct=0.252258 moved_to_replicas=215806 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] avg=131072.0000 max=132384 max_rank=3 min=123905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=117681 max_load_reduction_pct=0.470602 moved_to_replicas=346542 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[103631, 97885, 123905, 158904, 160957, 250065, 120411, 32818] avg=131072.0000 max=250065 max_rank=5 min=32818 min_rank=7 imbalance_ratio=1.907845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[131620, 103928, 175808, 65993, 131050, 131027, 160939, 148211] avg=131072.0000 max=175808 max_rank=2 min=65993 min_rank=3 imbalance_ratio=1.341309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[70429, 98344, 164444, 162459, 102125, 196446, 190311, 64018] avg=131072.0000 max=196446 max_rank=5 min=64018 min_rank=7 imbalance_ratio=1.498764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] avg=131072.0000 max=132384 max_rank=3 min=123905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=117681 max_load_reduction_pct=0.470602 moved_to_replicas=346542 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125346 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=43423 max_load_reduction_pct=0.246991 moved_to_replicas=176291 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[103631, 97885, 123905, 158904, 160957, 250065, 120411, 32818] avg=131072.0000 max=250065 max_rank=5 min=32818 min_rank=7 imbalance_ratio=1.907845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] avg=131072.0000 max=132384 max_rank=3 min=123905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=117681 max_load_reduction_pct=0.470602 moved_to_replicas=346542 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125346 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=43423 max_load_reduction_pct=0.246991 moved_to_replicas=176291 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[131620, 103928, 175808, 65993, 131050, 131027, 160939, 148211] avg=131072.0000 max=175808 max_rank=2 min=65993 min_rank=3 imbalance_ratio=1.341309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[70429, 98344, 164444, 162459, 102125, 196446, 190311, 64018] avg=131072.0000 max=196446 max_rank=5 min=64018 min_rank=7 imbalance_ratio=1.498764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] avg=131072.0000 max=132384 max_rank=6 min=124727 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64062 max_load_reduction_pct=0.326105 moved_to_replicas=367358 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] avg=131072.0000 max=132384 max_rank=3 min=123905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=117681 max_load_reduction_pct=0.470602 moved_to_replicas=346542 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[70429, 98344, 164444, 162459, 102125, 196446, 190311, 64018] avg=131072.0000 max=196446 max_rank=5 min=64018 min_rank=7 imbalance_ratio=1.498764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[103631, 97885, 123905, 158904, 160957, 250065, 120411, 32818] avg=131072.0000 max=250065 max_rank=5 min=32818 min_rank=7 imbalance_ratio=1.907845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] avg=131072.0000 max=132384 max_rank=6 min=124727 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64062 max_load_reduction_pct=0.326105 moved_to_replicas=367358 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[131620, 132383, 132382, 125346, 131050, 131027, 132383, 132385] avg=131072.0000 max=132385 max_rank=7 min=125346 min_rank=3 imbalance_ratio=1.010017 max_load_reduction=43423 max_load_reduction_pct=0.246991 moved_to_replicas=176291 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[70429, 98344, 164444, 162459, 102125, 196446, 190311, 64018] avg=131072.0000 max=196446 max_rank=5 min=64018 min_rank=7 imbalance_ratio=1.498764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[103631, 97885, 123905, 158904, 160957, 250065, 120411, 32818] avg=131072.0000 max=250065 max_rank=5 min=32818 min_rank=7 imbalance_ratio=1.907845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[69039, 99316, 222123, 98289, 65795, 167361, 97302, 229351] avg=131072.0000 max=229351 max_rank=7 min=65795 min_rank=4 imbalance_ratio=1.749809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] avg=131072.0000 max=132384 max_rank=6 min=124727 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64062 max_load_reduction_pct=0.326105 moved_to_replicas=367358 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[70429, 98344, 164444, 162459, 102125, 196446, 190311, 64018] avg=131072.0000 max=196446 max_rank=5 min=64018 min_rank=7 imbalance_ratio=1.498764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] avg=131072.0000 max=132384 max_rank=3 min=123905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=117681 max_load_reduction_pct=0.470602 moved_to_replicas=346542 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] avg=131072.0000 max=132384 max_rank=6 min=124727 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64062 max_load_reduction_pct=0.326105 moved_to_replicas=367358 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] avg=131072.0000 max=132384 max_rank=3 min=123905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=117681 max_load_reduction_pct=0.470602 moved_to_replicas=346542 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[69039, 99316, 222123, 98289, 65795, 167361, 97302, 229351] avg=131072.0000 max=229351 max_rank=7 min=65795 min_rank=4 imbalance_ratio=1.749809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] avg=131072.0000 max=132384 max_rank=2 min=127927 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96967 max_load_reduction_pct=0.422789 moved_to_replicas=441915 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[103631, 97885, 123905, 158904, 160957, 250065, 120411, 32818] avg=131072.0000 max=250065 max_rank=5 min=32818 min_rank=7 imbalance_ratio=1.907845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] avg=131072.0000 max=132384 max_rank=6 min=124727 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64062 max_load_reduction_pct=0.326105 moved_to_replicas=367358 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[69039, 99316, 222123, 98289, 65795, 167361, 97302, 229351] avg=131072.0000 max=229351 max_rank=7 min=65795 min_rank=4 imbalance_ratio=1.749809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[70429, 98344, 164444, 162459, 102125, 196446, 190311, 64018] avg=131072.0000 max=196446 max_rank=5 min=64018 min_rank=7 imbalance_ratio=1.498764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[69039, 99316, 222123, 98289, 65795, 167361, 97302, 229351] avg=131072.0000 max=229351 max_rank=7 min=65795 min_rank=4 imbalance_ratio=1.749809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] avg=131072.0000 max=132384 max_rank=2 min=127927 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96967 max_load_reduction_pct=0.422789 moved_to_replicas=441915 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[132205, 130651, 123905, 132384, 132383, 132384, 132282, 132382] avg=131072.0000 max=132384 max_rank=3 min=123905 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=117681 max_load_reduction_pct=0.470602 moved_to_replicas=346542 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] avg=131072.0000 max=132384 max_rank=2 min=127927 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96967 max_load_reduction_pct=0.422789 moved_to_replicas=441915 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[106362, 130342, 107091, 129272, 130399, 124569, 154763, 165778] avg=131072.0000 max=165778 max_rank=7 min=106362 min_rank=0 imbalance_ratio=1.264786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[70429, 98344, 164444, 162459, 102125, 196446, 190311, 64018] avg=131072.0000 max=196446 max_rank=5 min=64018 min_rank=7 imbalance_ratio=1.498764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[69039, 99316, 222123, 98289, 65795, 167361, 97302, 229351] avg=131072.0000 max=229351 max_rank=7 min=65795 min_rank=4 imbalance_ratio=1.749809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] avg=131072.0000 max=132384 max_rank=2 min=127927 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96967 max_load_reduction_pct=0.422789 moved_to_replicas=441915 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] avg=131072.0000 max=132384 max_rank=6 min=124727 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64062 max_load_reduction_pct=0.326105 moved_to_replicas=367358 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[106362, 130342, 107091, 129272, 130399, 124569, 154763, 165778] avg=131072.0000 max=165778 max_rank=7 min=106362 min_rank=0 imbalance_ratio=1.264786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] avg=131072.0000 max=132387 max_rank=2 min=129030 min_rank=5 imbalance_ratio=1.010033 max_load_reduction=33391 max_load_reduction_pct=0.201420 moved_to_replicas=104362 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[70429, 98344, 164444, 162459, 102125, 196446, 190311, 64018] avg=131072.0000 max=196446 max_rank=5 min=64018 min_rank=7 imbalance_ratio=1.498764 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] avg=131072.0000 max=132384 max_rank=6 min=124727 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64062 max_load_reduction_pct=0.326105 moved_to_replicas=367358 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] avg=131072.0000 max=132384 max_rank=2 min=127927 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96967 max_load_reduction_pct=0.422789 moved_to_replicas=441915 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[106362, 130342, 107091, 129272, 130399, 124569, 154763, 165778] avg=131072.0000 max=165778 max_rank=7 min=106362 min_rank=0 imbalance_ratio=1.264786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] avg=131072.0000 max=132387 max_rank=2 min=129030 min_rank=5 imbalance_ratio=1.010033 max_load_reduction=33391 max_load_reduction_pct=0.201420 moved_to_replicas=104362 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[106362, 130342, 107091, 129272, 130399, 124569, 154763, 165778] avg=131072.0000 max=165778 max_rank=7 min=106362 min_rank=0 imbalance_ratio=1.264786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[69039, 99316, 222123, 98289, 65795, 167361, 97302, 229351] avg=131072.0000 max=229351 max_rank=7 min=65795 min_rank=4 imbalance_ratio=1.749809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[132383, 124727, 132383, 132383, 132383, 132383, 132384, 129550] avg=131072.0000 max=132384 max_rank=6 min=124727 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64062 max_load_reduction_pct=0.326105 moved_to_replicas=367358 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] avg=131072.0000 max=132387 max_rank=2 min=129030 min_rank=5 imbalance_ratio=1.010033 max_load_reduction=33391 max_load_reduction_pct=0.201420 moved_to_replicas=104362 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[144947, 76503, 139442, 98501, 33299, 164026, 162710, 229148] avg=131072.0000 max=229148 max_rank=7 min=33299 min_rank=4 imbalance_ratio=1.748260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[69039, 99316, 222123, 98289, 65795, 167361, 97302, 229351] avg=131072.0000 max=229351 max_rank=7 min=65795 min_rank=4 imbalance_ratio=1.749809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[106362, 130342, 107091, 129272, 130399, 124569, 154763, 165778] avg=131072.0000 max=165778 max_rank=7 min=106362 min_rank=0 imbalance_ratio=1.264786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] avg=131072.0000 max=132387 max_rank=2 min=129030 min_rank=5 imbalance_ratio=1.010033 max_load_reduction=33391 max_load_reduction_pct=0.201420 moved_to_replicas=104362 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] avg=131072.0000 max=132384 max_rank=2 min=127927 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96967 max_load_reduction_pct=0.422789 moved_to_replicas=441915 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[144947, 76503, 139442, 98501, 33299, 164026, 162710, 229148] avg=131072.0000 max=229148 max_rank=7 min=33299 min_rank=4 imbalance_ratio=1.748260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] avg=131072.0000 max=132385 max_rank=0 min=127359 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=96763 max_load_reduction_pct=0.422273 moved_to_replicas=357313 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[69039, 99316, 222123, 98289, 65795, 167361, 97302, 229351] avg=131072.0000 max=229351 max_rank=7 min=65795 min_rank=4 imbalance_ratio=1.749809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] avg=131072.0000 max=132384 max_rank=2 min=127927 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96967 max_load_reduction_pct=0.422789 moved_to_replicas=441915 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] avg=131072.0000 max=132387 max_rank=2 min=129030 min_rank=5 imbalance_ratio=1.010033 max_load_reduction=33391 max_load_reduction_pct=0.201420 moved_to_replicas=104362 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[144947, 76503, 139442, 98501, 33299, 164026, 162710, 229148] avg=131072.0000 max=229148 max_rank=7 min=33299 min_rank=4 imbalance_ratio=1.748260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] avg=131072.0000 max=132385 max_rank=0 min=127359 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=96763 max_load_reduction_pct=0.422273 moved_to_replicas=357313 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[144947, 76503, 139442, 98501, 33299, 164026, 162710, 229148] avg=131072.0000 max=229148 max_rank=7 min=33299 min_rank=4 imbalance_ratio=1.748260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[106362, 130342, 107091, 129272, 130399, 124569, 154763, 165778] avg=131072.0000 max=165778 max_rank=7 min=106362 min_rank=0 imbalance_ratio=1.264786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[132383, 127927, 132384, 131054, 131324, 132383, 128738, 132383] avg=131072.0000 max=132384 max_rank=2 min=127927 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=96967 max_load_reduction_pct=0.422789 moved_to_replicas=441915 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[62319, 210216, 63915, 163967, 67529, 198913, 97245, 184472] avg=131072.0000 max=210216 max_rank=1 min=62319 min_rank=0 imbalance_ratio=1.603821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] avg=131072.0000 max=132385 max_rank=0 min=127359 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=96763 max_load_reduction_pct=0.422273 moved_to_replicas=357313 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[106362, 130342, 107091, 129272, 130399, 124569, 154763, 165778] avg=131072.0000 max=165778 max_rank=7 min=106362 min_rank=0 imbalance_ratio=1.264786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] avg=131072.0000 max=132385 max_rank=0 min=127359 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=96763 max_load_reduction_pct=0.422273 moved_to_replicas=357313 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[144947, 76503, 139442, 98501, 33299, 164026, 162710, 229148] avg=131072.0000 max=229148 max_rank=7 min=33299 min_rank=4 imbalance_ratio=1.748260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] avg=131072.0000 max=132387 max_rank=2 min=129030 min_rank=5 imbalance_ratio=1.010033 max_load_reduction=33391 max_load_reduction_pct=0.201420 moved_to_replicas=104362 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[62319, 210216, 63915, 163967, 67529, 198913, 97245, 184472] avg=131072.0000 max=210216 max_rank=1 min=62319 min_rank=0 imbalance_ratio=1.603821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] avg=131072.0000 max=132383 max_rank=3 min=127850 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=77833 max_load_reduction_pct=0.370253 moved_to_replicas=460054 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[106362, 130342, 107091, 129272, 130399, 124569, 154763, 165778] avg=131072.0000 max=165778 max_rank=7 min=106362 min_rank=0 imbalance_ratio=1.264786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] avg=131072.0000 max=132387 max_rank=2 min=129030 min_rank=5 imbalance_ratio=1.010033 max_load_reduction=33391 max_load_reduction_pct=0.201420 moved_to_replicas=104362 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[62319, 210216, 63915, 163967, 67529, 198913, 97245, 184472] avg=131072.0000 max=210216 max_rank=1 min=62319 min_rank=0 imbalance_ratio=1.603821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] avg=131072.0000 max=132385 max_rank=0 min=127359 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=96763 max_load_reduction_pct=0.422273 moved_to_replicas=357313 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[62319, 210216, 63915, 163967, 67529, 198913, 97245, 184472] avg=131072.0000 max=210216 max_rank=1 min=62319 min_rank=0 imbalance_ratio=1.603821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] avg=131072.0000 max=132383 max_rank=3 min=127850 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=77833 max_load_reduction_pct=0.370253 moved_to_replicas=460054 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[132381, 130342, 132387, 129272, 130399, 129030, 132382, 132383] avg=131072.0000 max=132387 max_rank=2 min=129030 min_rank=5 imbalance_ratio=1.010033 max_load_reduction=33391 max_load_reduction_pct=0.201420 moved_to_replicas=104362 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[196891, 163714, 98389, 163474, 98178, 163830, 98407, 65693] avg=131072.0000 max=196891 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.502159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[144947, 76503, 139442, 98501, 33299, 164026, 162710, 229148] avg=131072.0000 max=229148 max_rank=7 min=33299 min_rank=4 imbalance_ratio=1.748260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] avg=131072.0000 max=132383 max_rank=3 min=127850 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=77833 max_load_reduction_pct=0.370253 moved_to_replicas=460054 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[144947, 76503, 139442, 98501, 33299, 164026, 162710, 229148] avg=131072.0000 max=229148 max_rank=7 min=33299 min_rank=4 imbalance_ratio=1.748260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] avg=131072.0000 max=132383 max_rank=3 min=127850 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=77833 max_load_reduction_pct=0.370253 moved_to_replicas=460054 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[62319, 210216, 63915, 163967, 67529, 198913, 97245, 184472] avg=131072.0000 max=210216 max_rank=1 min=62319 min_rank=0 imbalance_ratio=1.603821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] avg=131072.0000 max=132383 max_rank=0 min=129550 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64508 max_load_reduction_pct=0.327633 moved_to_replicas=316820 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[196891, 163714, 98389, 163474, 98178, 163830, 98407, 65693] avg=131072.0000 max=196891 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.502159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] avg=131072.0000 max=132385 max_rank=0 min=127359 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=96763 max_load_reduction_pct=0.422273 moved_to_replicas=357313 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[144947, 76503, 139442, 98501, 33299, 164026, 162710, 229148] avg=131072.0000 max=229148 max_rank=7 min=33299 min_rank=4 imbalance_ratio=1.748260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] avg=131072.0000 max=132385 max_rank=0 min=127359 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=96763 max_load_reduction_pct=0.422273 moved_to_replicas=357313 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[196891, 163714, 98389, 163474, 98178, 163830, 98407, 65693] avg=131072.0000 max=196891 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.502159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] avg=131072.0000 max=132383 max_rank=3 min=127850 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=77833 max_load_reduction_pct=0.370253 moved_to_replicas=460054 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] avg=131072.0000 max=132383 max_rank=0 min=129550 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64508 max_load_reduction_pct=0.327633 moved_to_replicas=316820 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[196891, 163714, 98389, 163474, 98178, 163830, 98407, 65693] avg=131072.0000 max=196891 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.502159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[132385, 127359, 132382, 130144, 129157, 132383, 132383, 132383] avg=131072.0000 max=132385 max_rank=0 min=127359 min_rank=1 imbalance_ratio=1.010017 max_load_reduction=96763 max_load_reduction_pct=0.422273 moved_to_replicas=357313 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[134179, 229345, 20620, 163760, 104309, 134598, 163753, 98012] avg=131072.0000 max=229345 max_rank=1 min=20620 min_rank=2 imbalance_ratio=1.749763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] avg=131072.0000 max=132383 max_rank=0 min=129550 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64508 max_load_reduction_pct=0.327633 moved_to_replicas=316820 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[62319, 210216, 63915, 163967, 67529, 198913, 97245, 184472] avg=131072.0000 max=210216 max_rank=1 min=62319 min_rank=0 imbalance_ratio=1.603821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] avg=131072.0000 max=132383 max_rank=0 min=129550 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64508 max_load_reduction_pct=0.327633 moved_to_replicas=316820 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[62319, 210216, 63915, 163967, 67529, 198913, 97245, 184472] avg=131072.0000 max=210216 max_rank=1 min=62319 min_rank=0 imbalance_ratio=1.603821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] avg=131072.0000 max=132383 max_rank=0 min=124889 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96962 max_load_reduction_pct=0.422778 moved_to_replicas=328642 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[134179, 229345, 20620, 163760, 104309, 134598, 163753, 98012] avg=131072.0000 max=229345 max_rank=1 min=20620 min_rank=2 imbalance_ratio=1.749763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[196891, 163714, 98389, 163474, 98178, 163830, 98407, 65693] avg=131072.0000 max=196891 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.502159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] avg=131072.0000 max=132383 max_rank=3 min=127850 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=77833 max_load_reduction_pct=0.370253 moved_to_replicas=460054 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[62319, 210216, 63915, 163967, 67529, 198913, 97245, 184472] avg=131072.0000 max=210216 max_rank=1 min=62319 min_rank=0 imbalance_ratio=1.603821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] avg=131072.0000 max=132383 max_rank=3 min=127850 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=77833 max_load_reduction_pct=0.370253 moved_to_replicas=460054 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[134179, 229345, 20620, 163760, 104309, 134598, 163753, 98012] avg=131072.0000 max=229345 max_rank=1 min=20620 min_rank=2 imbalance_ratio=1.749763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] avg=131072.0000 max=132383 max_rank=0 min=124889 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96962 max_load_reduction_pct=0.422778 moved_to_replicas=328642 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[134179, 229345, 20620, 163760, 104309, 134598, 163753, 98012] avg=131072.0000 max=229345 max_rank=1 min=20620 min_rank=2 imbalance_ratio=1.749763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] avg=131072.0000 max=132383 max_rank=0 min=129550 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64508 max_load_reduction_pct=0.327633 moved_to_replicas=316820 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[127850, 132382, 128801, 132383, 132383, 132383, 130011, 132383] avg=131072.0000 max=132383 max_rank=3 min=127850 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=77833 max_load_reduction_pct=0.370253 moved_to_replicas=460054 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[65520, 169214, 66474, 164987, 93744, 228345, 186507, 73785] avg=131072.0000 max=228345 max_rank=5 min=65520 min_rank=0 imbalance_ratio=1.742134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] avg=131072.0000 max=132383 max_rank=0 min=124889 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96962 max_load_reduction_pct=0.422778 moved_to_replicas=328642 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[196891, 163714, 98389, 163474, 98178, 163830, 98407, 65693] avg=131072.0000 max=196891 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.502159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] avg=131072.0000 max=132383 max_rank=0 min=124889 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96962 max_load_reduction_pct=0.422778 moved_to_replicas=328642 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[196891, 163714, 98389, 163474, 98178, 163830, 98407, 65693] avg=131072.0000 max=196891 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.502159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126505 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95961 max_load_reduction_pct=0.420246 moved_to_replicas=440657 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[65520, 169214, 66474, 164987, 93744, 228345, 186507, 73785] avg=131072.0000 max=228345 max_rank=5 min=65520 min_rank=0 imbalance_ratio=1.742134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[134179, 229345, 20620, 163760, 104309, 134598, 163753, 98012] avg=131072.0000 max=229345 max_rank=1 min=20620 min_rank=2 imbalance_ratio=1.749763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] avg=131072.0000 max=132383 max_rank=0 min=129550 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64508 max_load_reduction_pct=0.327633 moved_to_replicas=316820 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[196891, 163714, 98389, 163474, 98178, 163830, 98407, 65693] avg=131072.0000 max=196891 max_rank=0 min=65693 min_rank=7 imbalance_ratio=1.502159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[65520, 169214, 66474, 164987, 93744, 228345, 186507, 73785] avg=131072.0000 max=228345 max_rank=5 min=65520 min_rank=0 imbalance_ratio=1.742134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] avg=131072.0000 max=132383 max_rank=0 min=129550 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64508 max_load_reduction_pct=0.327633 moved_to_replicas=316820 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126505 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95961 max_load_reduction_pct=0.420246 moved_to_replicas=440657 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[65520, 169214, 66474, 164987, 93744, 228345, 186507, 73785] avg=131072.0000 max=228345 max_rank=5 min=65520 min_rank=0 imbalance_ratio=1.742134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] avg=131072.0000 max=132383 max_rank=0 min=124889 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96962 max_load_reduction_pct=0.422778 moved_to_replicas=328642 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[130872, 98323, 94951, 67786, 132457, 196768, 229329, 98090] avg=131072.0000 max=229329 max_rank=6 min=67786 min_rank=3 imbalance_ratio=1.749641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[132383, 132383, 129836, 132383, 129920, 132383, 129738, 129550] avg=131072.0000 max=132383 max_rank=0 min=129550 min_rank=7 imbalance_ratio=1.010002 max_load_reduction=64508 max_load_reduction_pct=0.327633 moved_to_replicas=316820 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126505 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95961 max_load_reduction_pct=0.420246 moved_to_replicas=440657 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[134179, 229345, 20620, 163760, 104309, 134598, 163753, 98012] avg=131072.0000 max=229345 max_rank=1 min=20620 min_rank=2 imbalance_ratio=1.749763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126505 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95961 max_load_reduction_pct=0.420246 moved_to_replicas=440657 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] avg=131072.0000 max=132383 max_rank=4 min=127791 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96946 max_load_reduction_pct=0.422738 moved_to_replicas=322658 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[134179, 229345, 20620, 163760, 104309, 134598, 163753, 98012] avg=131072.0000 max=229345 max_rank=1 min=20620 min_rank=2 imbalance_ratio=1.749763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[130872, 98323, 94951, 67786, 132457, 196768, 229329, 98090] avg=131072.0000 max=229329 max_rank=6 min=67786 min_rank=3 imbalance_ratio=1.749641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[65520, 169214, 66474, 164987, 93744, 228345, 186507, 73785] avg=131072.0000 max=228345 max_rank=5 min=65520 min_rank=0 imbalance_ratio=1.742134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] avg=131072.0000 max=132383 max_rank=0 min=124889 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96962 max_load_reduction_pct=0.422778 moved_to_replicas=328642 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[134179, 229345, 20620, 163760, 104309, 134598, 163753, 98012] avg=131072.0000 max=229345 max_rank=1 min=20620 min_rank=2 imbalance_ratio=1.749763 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[130872, 98323, 94951, 67786, 132457, 196768, 229329, 98090] avg=131072.0000 max=229329 max_rank=6 min=67786 min_rank=3 imbalance_ratio=1.749641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] avg=131072.0000 max=132383 max_rank=4 min=127791 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96946 max_load_reduction_pct=0.422738 moved_to_replicas=322658 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] avg=131072.0000 max=132383 max_rank=0 min=124889 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96962 max_load_reduction_pct=0.422778 moved_to_replicas=328642 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[130872, 98323, 94951, 67786, 132457, 196768, 229329, 98090] avg=131072.0000 max=229329 max_rank=6 min=67786 min_rank=3 imbalance_ratio=1.749641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126505 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95961 max_load_reduction_pct=0.420246 moved_to_replicas=440657 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[32953, 196136, 65058, 131360, 65104, 225369, 135542, 197054] avg=131072.0000 max=225369 max_rank=5 min=32953 min_rank=0 imbalance_ratio=1.719429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] avg=131072.0000 max=132383 max_rank=4 min=127791 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96946 max_load_reduction_pct=0.422738 moved_to_replicas=322658 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[132383, 132383, 124889, 132383, 132383, 132383, 132383, 129389] avg=131072.0000 max=132383 max_rank=0 min=124889 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96962 max_load_reduction_pct=0.422778 moved_to_replicas=328642 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[65520, 169214, 66474, 164987, 93744, 228345, 186507, 73785] avg=131072.0000 max=228345 max_rank=5 min=65520 min_rank=0 imbalance_ratio=1.742134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] avg=131072.0000 max=132383 max_rank=4 min=127791 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96946 max_load_reduction_pct=0.422738 moved_to_replicas=322658 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127629 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=92986 max_load_reduction_pct=0.412594 moved_to_replicas=451147 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[32953, 196136, 65058, 131360, 65104, 225369, 135542, 197054] avg=131072.0000 max=225369 max_rank=5 min=32953 min_rank=0 imbalance_ratio=1.719429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[65520, 169214, 66474, 164987, 93744, 228345, 186507, 73785] avg=131072.0000 max=228345 max_rank=5 min=65520 min_rank=0 imbalance_ratio=1.742134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[130872, 98323, 94951, 67786, 132457, 196768, 229329, 98090] avg=131072.0000 max=229329 max_rank=6 min=67786 min_rank=3 imbalance_ratio=1.749641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126505 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95961 max_load_reduction_pct=0.420246 moved_to_replicas=440657 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[32953, 196136, 65058, 131360, 65104, 225369, 135542, 197054] avg=131072.0000 max=225369 max_rank=5 min=32953 min_rank=0 imbalance_ratio=1.719429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[65520, 169214, 66474, 164987, 93744, 228345, 186507, 73785] avg=131072.0000 max=228345 max_rank=5 min=65520 min_rank=0 imbalance_ratio=1.742134 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127629 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=92986 max_load_reduction_pct=0.412594 moved_to_replicas=451147 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[32953, 196136, 65058, 131360, 65104, 225369, 135542, 197054] avg=131072.0000 max=225369 max_rank=5 min=32953 min_rank=0 imbalance_ratio=1.719429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126505 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95961 max_load_reduction_pct=0.420246 moved_to_replicas=440657 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] avg=131072.0000 max=132383 max_rank=4 min=127791 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96946 max_load_reduction_pct=0.422738 moved_to_replicas=322658 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127629 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=92986 max_load_reduction_pct=0.412594 moved_to_replicas=451147 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[128152, 132383, 132006, 132383, 126505, 132383, 132380, 132384] avg=131072.0000 max=132384 max_rank=7 min=126505 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=95961 max_load_reduction_pct=0.420246 moved_to_replicas=440657 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[130872, 98323, 94951, 67786, 132457, 196768, 229329, 98090] avg=131072.0000 max=229329 max_rank=6 min=67786 min_rank=3 imbalance_ratio=1.749641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127629 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=92986 max_load_reduction_pct=0.412594 moved_to_replicas=451147 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[130872, 98323, 94951, 67786, 132457, 196768, 229329, 98090] avg=131072.0000 max=229329 max_rank=6 min=67786 min_rank=3 imbalance_ratio=1.749641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[32953, 196136, 65058, 131360, 65104, 225369, 135542, 197054] avg=131072.0000 max=225369 max_rank=5 min=32953 min_rank=0 imbalance_ratio=1.719429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] avg=131072.0000 max=132383 max_rank=4 min=127791 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96946 max_load_reduction_pct=0.422738 moved_to_replicas=322658 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[130872, 98323, 94951, 67786, 132457, 196768, 229329, 98090] avg=131072.0000 max=229329 max_rank=6 min=67786 min_rank=3 imbalance_ratio=1.749641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] avg=131072.0000 max=132383 max_rank=4 min=127791 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96946 max_load_reduction_pct=0.422738 moved_to_replicas=322658 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127629 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=92986 max_load_reduction_pct=0.412594 moved_to_replicas=451147 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[130872, 131088, 127791, 132172, 132383, 132383, 132383, 129504] avg=131072.0000 max=132383 max_rank=4 min=127791 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=96946 max_load_reduction_pct=0.422738 moved_to_replicas=322658 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[32953, 196136, 65058, 131360, 65104, 225369, 135542, 197054] avg=131072.0000 max=225369 max_rank=5 min=32953 min_rank=0 imbalance_ratio=1.719429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[32953, 196136, 65058, 131360, 65104, 225369, 135542, 197054] avg=131072.0000 max=225369 max_rank=5 min=32953 min_rank=0 imbalance_ratio=1.719429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127629 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=92986 max_load_reduction_pct=0.412594 moved_to_replicas=451147 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[32953, 196136, 65058, 131360, 65104, 225369, 135542, 197054] avg=131072.0000 max=225369 max_rank=5 min=32953 min_rank=0 imbalance_ratio=1.719429 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127629 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=92986 max_load_reduction_pct=0.412594 moved_to_replicas=451147 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[129471, 132383, 130584, 131360, 127629, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=127629 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=92986 max_load_reduction_pct=0.412594 moved_to_replicas=451147 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=4202273 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1004928 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=4201077 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1005824 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=4161763 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1043200 peak_demand=132387 required_capacity=1.010033 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=4215779 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=988928 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=4193586 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016064 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=4206374 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=999680 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=4180752 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1024768 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=4192828 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015168 peak_demand=132385 required_capacity=1.010017 + [2026-08-30 05:17:57] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 581.0 | throughput per GPU (TFLOP/s/GPU): 573.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.806993E+00 | loss scale: 1.0 | grad norm: 4.966 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=163840 expected_recv=259134 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=163840 expected_recv=260417 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=163840 expected_recv=261555 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=163840 expected_recv=256981 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=163840 expected_recv=259134 actual_valid_recv=261504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=163840 expected_recv=260417 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=163840 expected_recv=261555 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=163840 expected_recv=256981 actual_valid_recv=259712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=261504 slack_slots=66176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=259712 slack_slots=67968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=163840 expected_recv=252433 expected_overflow=0 exact_recv_per_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=163840 expected_recv=261714 expected_overflow=0 exact_recv_per_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=163840 expected_recv=252433 actual_valid_recv=255488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=163840 expected_recv=261714 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=255488 slack_slots=72192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=163840 expected_recv=259175 expected_overflow=0 exact_recv_per_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=268032 slack_slots=59648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=163840 expected_recv=259175 actual_valid_recv=262400 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=268032 slack_slots=59648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=262400 slack_slots=65280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=163840 expected_recv=258363 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=163840 expected_recv=260591 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=163840 expected_recv=258363 actual_valid_recv=260864 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=163840 expected_recv=260591 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=163840 expected_recv=254364 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=260864 slack_slots=66816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267904 slack_slots=59776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=163840 expected_recv=254364 actual_valid_recv=257664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=257664 slack_slots=70016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267904 slack_slots=59776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=163840 expected_recv=254277 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=163840 expected_recv=254278 expected_overflow=0 exact_recv_per_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=163840 expected_recv=254277 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=163840 expected_recv=254278 actual_valid_recv=257280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=267776 slack_slots=59904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=257408 slack_slots=70272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=163840 expected_recv=263745 expected_overflow=0 exact_recv_per_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=257280 slack_slots=70400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=163840 expected_recv=263745 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=163840 expected_recv=248081 expected_overflow=0 exact_recv_per_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=163840 expected_recv=261892 expected_overflow=0 exact_recv_per_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=163840 expected_recv=248081 actual_valid_recv=250880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=163840 expected_recv=264371 expected_overflow=0 exact_recv_per_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=163840 expected_recv=261892 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=250880 slack_slots=76800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=163840 expected_recv=264371 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=163840 expected_recv=263573 expected_overflow=0 exact_recv_per_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=163840 expected_recv=263573 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=163840 expected_recv=248800 expected_overflow=0 exact_recv_per_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=163840 expected_recv=248800 actual_valid_recv=251392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=163840 expected_recv=262032 expected_overflow=0 exact_recv_per_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=163840 expected_recv=263684 expected_overflow=0 exact_recv_per_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=251392 slack_slots=76288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=163840 expected_recv=262032 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=163840 expected_recv=263684 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=268032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=163840 expected_recv=256764 expected_overflow=0 exact_recv_per_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=268032 slack_slots=59648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=163840 expected_recv=256764 actual_valid_recv=259584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=259584 slack_slots=68096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=163840 expected_recv=262272 expected_overflow=0 exact_recv_per_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=163840 expected_recv=263588 expected_overflow=0 exact_recv_per_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=163840 expected_recv=264769 expected_overflow=0 exact_recv_per_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=163840 expected_recv=262272 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=163840 expected_recv=255461 expected_overflow=0 exact_recv_per_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=163840 expected_recv=263588 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=163840 expected_recv=264769 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=163840 expected_recv=255461 actual_valid_recv=258432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=258432 slack_slots=69248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=163840 expected_recv=249770 expected_overflow=0 exact_recv_per_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=163840 expected_recv=259200 expected_overflow=0 exact_recv_per_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264353 expected_overflow=0 exact_recv_per_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=163840 expected_recv=249770 actual_valid_recv=253056 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=163840 expected_recv=259200 actual_valid_recv=261120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=253056 slack_slots=74624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264353 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=163840 expected_recv=259659 expected_overflow=0 exact_recv_per_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=261120 slack_slots=66560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=266624 slack_slots=61056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=163840 expected_recv=259659 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=163840 expected_recv=261058 expected_overflow=0 exact_recv_per_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=163840 expected_recv=259068 expected_overflow=0 exact_recv_per_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=163840 expected_recv=261058 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=163840 expected_recv=259068 actual_valid_recv=261888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=266368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=163840 expected_recv=259553 expected_overflow=0 exact_recv_per_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=261888 slack_slots=65792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266368 slack_slots=61312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=163840 expected_recv=259553 actual_valid_recv=261632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=163840 expected_recv=263517 expected_overflow=0 exact_recv_per_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=261632 slack_slots=66048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=163840 expected_recv=263517 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=163840 expected_recv=255935 expected_overflow=0 exact_recv_per_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=163840 expected_recv=255935 actual_valid_recv=258304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=163840 expected_recv=262542 expected_overflow=0 exact_recv_per_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=258304 slack_slots=69376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=163840 expected_recv=262542 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=163840 expected_recv=260355 expected_overflow=0 exact_recv_per_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=163840 expected_recv=259256 expected_overflow=0 exact_recv_per_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=163840 expected_recv=260355 actual_valid_recv=262144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=262144 slack_slots=65536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=163840 expected_recv=259256 actual_valid_recv=262272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=163840 expected_recv=264397 expected_overflow=0 exact_recv_per_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=163840 expected_recv=258446 expected_overflow=0 exact_recv_per_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=262272 slack_slots=65408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=163840 expected_recv=264397 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=163840 expected_recv=258446 actual_valid_recv=260736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=260736 slack_slots=66944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=163840 expected_recv=255438 expected_overflow=0 exact_recv_per_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=163840 expected_recv=259807 expected_overflow=0 exact_recv_per_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=163840 expected_recv=255438 actual_valid_recv=257920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=163840 expected_recv=259807 actual_valid_recv=262016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=257920 slack_slots=69760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=262016 slack_slots=65664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=163840 expected_recv=260712 expected_overflow=0 exact_recv_per_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=163840 expected_recv=261111 expected_overflow=0 exact_recv_per_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=267008 slack_slots=60672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=163840 expected_recv=261395 expected_overflow=0 exact_recv_per_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=163840 expected_recv=260712 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=163840 expected_recv=261111 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=163840 expected_recv=254870 expected_overflow=0 exact_recv_per_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=163840 expected_recv=261395 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=163840 expected_recv=254870 actual_valid_recv=257408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=163840 expected_recv=255363 expected_overflow=0 exact_recv_per_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=257408 slack_slots=70272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=163840 expected_recv=255363 actual_valid_recv=257280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261423 expected_overflow=0 exact_recv_per_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=257280 slack_slots=70400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261423 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261084 expected_overflow=0 exact_recv_per_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=163840 expected_recv=260218 expected_overflow=0 exact_recv_per_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261084 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=163840 expected_recv=260218 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=163840 expected_recv=260764 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=163840 expected_recv=264767 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=267904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=163840 expected_recv=260764 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267904 slack_slots=59776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=163840 expected_recv=264767 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=267264 slack_slots=60416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=163840 expected_recv=260159 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=163840 expected_recv=256732 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=163840 expected_recv=260431 expected_overflow=0 exact_recv_per_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=163840 expected_recv=260159 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=163840 expected_recv=256732 actual_valid_recv=258944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=163840 expected_recv=260431 actual_valid_recv=262528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261589 expected_overflow=0 exact_recv_per_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=266752 slack_slots=60928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=258944 slack_slots=68736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=262528 slack_slots=65152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=163840 expected_recv=252855 expected_overflow=0 exact_recv_per_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261589 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=163840 expected_recv=264768 expected_overflow=0 exact_recv_per_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261995 expected_overflow=0 exact_recv_per_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=163840 expected_recv=252855 actual_valid_recv=255744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=163840 expected_recv=264768 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=255744 slack_slots=71936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261995 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=163840 expected_recv=264764 expected_overflow=0 exact_recv_per_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=163840 expected_recv=262024 expected_overflow=0 exact_recv_per_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=163840 expected_recv=264391 expected_overflow=0 exact_recv_per_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=163840 expected_recv=264764 actual_valid_recv=267392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=163840 expected_recv=262024 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=163840 expected_recv=264391 actual_valid_recv=266496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=163840 expected_recv=262643 expected_overflow=0 exact_recv_per_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=267392 slack_slots=60288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=163840 expected_recv=249202 expected_overflow=0 exact_recv_per_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266496 slack_slots=61184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=163840 expected_recv=262643 actual_valid_recv=265600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261478 expected_overflow=0 exact_recv_per_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=163840 expected_recv=249202 actual_valid_recv=251264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=265600 slack_slots=62080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=251264 slack_slots=76416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261478 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266880 slack_slots=60800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=163840 expected_recv=264765 expected_overflow=0 exact_recv_per_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=163840 expected_recv=264766 expected_overflow=0 exact_recv_per_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=266240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=163840 expected_recv=264765 actual_valid_recv=267520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267648 slack_slots=60032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=163840 expected_recv=264766 actual_valid_recv=267136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=266240 slack_slots=61440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267520 slack_slots=60160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[152661, 133015, 121803, 103850, 142781, 112502, 137801, 144163] avg=131072.0000 max=152661 max_rank=0 min=103850 min_rank=3 imbalance_ratio=1.164711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[152661, 133015, 121803, 103850, 142781, 112502, 137801, 144163] avg=131072.0000 max=152661 max_rank=0 min=103850 min_rank=3 imbalance_ratio=1.164711 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=267136 slack_slots=60544 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[152661, 133015, 121803, 103850, 142781, 112502, 137801, 144163] avg=131072.0000 max=152661 max_rank=0 min=103850 min_rank=3 imbalance_ratio=1.164711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[152661, 133015, 121803, 103850, 142781, 112502, 137801, 144163] avg=131072.0000 max=152661 max_rank=0 min=103850 min_rank=3 imbalance_ratio=1.164711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127853 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=20278 max_load_reduction_pct=0.132830 moved_to_replicas=97407 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127853 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=20278 max_load_reduction_pct=0.132830 moved_to_replicas=97407 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127853 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=20278 max_load_reduction_pct=0.132830 moved_to_replicas=97407 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[152661, 133015, 121803, 103850, 142781, 112502, 137801, 144163] avg=131072.0000 max=152661 max_rank=0 min=103850 min_rank=3 imbalance_ratio=1.164711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127853 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=20278 max_load_reduction_pct=0.132830 moved_to_replicas=97407 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[152661, 133015, 121803, 103850, 142781, 112502, 137801, 144163] avg=131072.0000 max=152661 max_rank=0 min=103850 min_rank=3 imbalance_ratio=1.164711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[152661, 133015, 121803, 103850, 142781, 112502, 137801, 144163] avg=131072.0000 max=152661 max_rank=0 min=103850 min_rank=3 imbalance_ratio=1.164711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[152661, 133015, 121803, 103850, 142781, 112502, 137801, 144163] avg=131072.0000 max=152661 max_rank=0 min=103850 min_rank=3 imbalance_ratio=1.164711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[91655, 97318, 160114, 167624, 89217, 129331, 212809, 100508] avg=131072.0000 max=212809 max_rank=6 min=89217 min_rank=4 imbalance_ratio=1.623604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[91655, 97318, 160114, 167624, 89217, 129331, 212809, 100508] avg=131072.0000 max=212809 max_rank=6 min=89217 min_rank=4 imbalance_ratio=1.623604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127853 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=20278 max_load_reduction_pct=0.132830 moved_to_replicas=97407 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127853 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=20278 max_load_reduction_pct=0.132830 moved_to_replicas=97407 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[91655, 97318, 160114, 167624, 89217, 129331, 212809, 100508] avg=131072.0000 max=212809 max_rank=6 min=89217 min_rank=4 imbalance_ratio=1.623604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127853 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=20278 max_load_reduction_pct=0.132830 moved_to_replicas=97407 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[132383, 132383, 127853, 130774, 132382, 128035, 132383, 132383] avg=131072.0000 max=132383 max_rank=0 min=127853 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=20278 max_load_reduction_pct=0.132830 moved_to_replicas=97407 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[91655, 97318, 160114, 167624, 89217, 129331, 212809, 100508] avg=131072.0000 max=212809 max_rank=6 min=89217 min_rank=4 imbalance_ratio=1.623604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124947 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=80426 max_load_reduction_pct=0.377926 moved_to_replicas=278803 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124947 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=80426 max_load_reduction_pct=0.377926 moved_to_replicas=278803 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124947 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=80426 max_load_reduction_pct=0.377926 moved_to_replicas=278803 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124947 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=80426 max_load_reduction_pct=0.377926 moved_to_replicas=278803 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[91655, 97318, 160114, 167624, 89217, 129331, 212809, 100508] avg=131072.0000 max=212809 max_rank=6 min=89217 min_rank=4 imbalance_ratio=1.623604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[91655, 97318, 160114, 167624, 89217, 129331, 212809, 100508] avg=131072.0000 max=212809 max_rank=6 min=89217 min_rank=4 imbalance_ratio=1.623604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[91655, 97318, 160114, 167624, 89217, 129331, 212809, 100508] avg=131072.0000 max=212809 max_rank=6 min=89217 min_rank=4 imbalance_ratio=1.623604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[91655, 97318, 160114, 167624, 89217, 129331, 212809, 100508] avg=131072.0000 max=212809 max_rank=6 min=89217 min_rank=4 imbalance_ratio=1.623604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[125127, 213082, 75726, 200775, 168252, 142052, 45369, 78193] avg=131072.0000 max=213082 max_rank=1 min=45369 min_rank=6 imbalance_ratio=1.625687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[125127, 213082, 75726, 200775, 168252, 142052, 45369, 78193] avg=131072.0000 max=213082 max_rank=1 min=45369 min_rank=6 imbalance_ratio=1.625687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124947 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=80426 max_load_reduction_pct=0.377926 moved_to_replicas=278803 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[125127, 213082, 75726, 200775, 168252, 142052, 45369, 78193] avg=131072.0000 max=213082 max_rank=1 min=45369 min_rank=6 imbalance_ratio=1.625687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124947 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=80426 max_load_reduction_pct=0.377926 moved_to_replicas=278803 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124947 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=80426 max_load_reduction_pct=0.377926 moved_to_replicas=278803 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[124947, 132383, 132383, 132383, 132383, 129331, 132383, 132383] avg=131072.0000 max=132383 max_rank=1 min=124947 min_rank=0 imbalance_ratio=1.010002 max_load_reduction=80426 max_load_reduction_pct=0.377926 moved_to_replicas=278803 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[125127, 213082, 75726, 200775, 168252, 142052, 45369, 78193] avg=131072.0000 max=213082 max_rank=1 min=45369 min_rank=6 imbalance_ratio=1.625687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] avg=131072.0000 max=132384 max_rank=5 min=126068 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=80698 max_load_reduction_pct=0.378718 moved_to_replicas=390198 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] avg=131072.0000 max=132384 max_rank=5 min=126068 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=80698 max_load_reduction_pct=0.378718 moved_to_replicas=390198 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] avg=131072.0000 max=132384 max_rank=5 min=126068 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=80698 max_load_reduction_pct=0.378718 moved_to_replicas=390198 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[125127, 213082, 75726, 200775, 168252, 142052, 45369, 78193] avg=131072.0000 max=213082 max_rank=1 min=45369 min_rank=6 imbalance_ratio=1.625687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] avg=131072.0000 max=132384 max_rank=5 min=126068 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=80698 max_load_reduction_pct=0.378718 moved_to_replicas=390198 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[125127, 213082, 75726, 200775, 168252, 142052, 45369, 78193] avg=131072.0000 max=213082 max_rank=1 min=45369 min_rank=6 imbalance_ratio=1.625687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[125127, 213082, 75726, 200775, 168252, 142052, 45369, 78193] avg=131072.0000 max=213082 max_rank=1 min=45369 min_rank=6 imbalance_ratio=1.625687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[125127, 213082, 75726, 200775, 168252, 142052, 45369, 78193] avg=131072.0000 max=213082 max_rank=1 min=45369 min_rank=6 imbalance_ratio=1.625687 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[147701, 87698, 135862, 194055, 118328, 151207, 150246, 63479] avg=131072.0000 max=194055 max_rank=3 min=63479 min_rank=7 imbalance_ratio=1.480522 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[147701, 87698, 135862, 194055, 118328, 151207, 150246, 63479] avg=131072.0000 max=194055 max_rank=3 min=63479 min_rank=7 imbalance_ratio=1.480522 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] avg=131072.0000 max=132384 max_rank=5 min=126068 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=80698 max_load_reduction_pct=0.378718 moved_to_replicas=390198 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[147701, 87698, 135862, 194055, 118328, 151207, 150246, 63479] avg=131072.0000 max=194055 max_rank=3 min=63479 min_rank=7 imbalance_ratio=1.480522 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] avg=131072.0000 max=132384 max_rank=5 min=126068 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=80698 max_load_reduction_pct=0.378718 moved_to_replicas=390198 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] avg=131072.0000 max=132384 max_rank=5 min=126068 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=80698 max_load_reduction_pct=0.378718 moved_to_replicas=390198 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[147701, 87698, 135862, 194055, 118328, 151207, 150246, 63479] avg=131072.0000 max=194055 max_rank=3 min=63479 min_rank=7 imbalance_ratio=1.480522 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[132382, 132383, 128209, 132383, 132383, 132384, 126068, 132384] avg=131072.0000 max=132384 max_rank=5 min=126068 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=80698 max_load_reduction_pct=0.378718 moved_to_replicas=390198 active_clone_links=22 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=61671 max_load_reduction_pct=0.317802 moved_to_replicas=233945 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=61671 max_load_reduction_pct=0.317802 moved_to_replicas=233945 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=61671 max_load_reduction_pct=0.317802 moved_to_replicas=233945 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=61671 max_load_reduction_pct=0.317802 moved_to_replicas=233945 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[147701, 87698, 135862, 194055, 118328, 151207, 150246, 63479] avg=131072.0000 max=194055 max_rank=3 min=63479 min_rank=7 imbalance_ratio=1.480522 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[147701, 87698, 135862, 194055, 118328, 151207, 150246, 63479] avg=131072.0000 max=194055 max_rank=3 min=63479 min_rank=7 imbalance_ratio=1.480522 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[147701, 87698, 135862, 194055, 118328, 151207, 150246, 63479] avg=131072.0000 max=194055 max_rank=3 min=63479 min_rank=7 imbalance_ratio=1.480522 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[147701, 87698, 135862, 194055, 118328, 151207, 150246, 63479] avg=131072.0000 max=194055 max_rank=3 min=63479 min_rank=7 imbalance_ratio=1.480522 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[67692, 130912, 189109, 112938, 163502, 100453, 161364, 122606] avg=131072.0000 max=189109 max_rank=2 min=67692 min_rank=0 imbalance_ratio=1.442787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[67692, 130912, 189109, 112938, 163502, 100453, 161364, 122606] avg=131072.0000 max=189109 max_rank=2 min=67692 min_rank=0 imbalance_ratio=1.442787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[67692, 130912, 189109, 112938, 163502, 100453, 161364, 122606] avg=131072.0000 max=189109 max_rank=2 min=67692 min_rank=0 imbalance_ratio=1.442787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=61671 max_load_reduction_pct=0.317802 moved_to_replicas=233945 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[67692, 130912, 189109, 112938, 163502, 100453, 161364, 122606] avg=131072.0000 max=189109 max_rank=2 min=67692 min_rank=0 imbalance_ratio=1.442787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=61671 max_load_reduction_pct=0.317802 moved_to_replicas=233945 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=61671 max_load_reduction_pct=0.317802 moved_to_replicas=233945 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] avg=131072.0000 max=132383 max_rank=2 min=124416 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=56726 max_load_reduction_pct=0.299965 moved_to_replicas=235734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] avg=131072.0000 max=132383 max_rank=2 min=124416 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=56726 max_load_reduction_pct=0.299965 moved_to_replicas=235734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[132382, 132384, 132383, 132383, 121895, 132383, 132383, 132383] avg=131072.0000 max=132384 max_rank=1 min=121895 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=61671 max_load_reduction_pct=0.317802 moved_to_replicas=233945 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] avg=131072.0000 max=132383 max_rank=2 min=124416 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=56726 max_load_reduction_pct=0.299965 moved_to_replicas=235734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] avg=131072.0000 max=132383 max_rank=2 min=124416 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=56726 max_load_reduction_pct=0.299965 moved_to_replicas=235734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[67692, 130912, 189109, 112938, 163502, 100453, 161364, 122606] avg=131072.0000 max=189109 max_rank=2 min=67692 min_rank=0 imbalance_ratio=1.442787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[67692, 130912, 189109, 112938, 163502, 100453, 161364, 122606] avg=131072.0000 max=189109 max_rank=2 min=67692 min_rank=0 imbalance_ratio=1.442787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[67692, 130912, 189109, 112938, 163502, 100453, 161364, 122606] avg=131072.0000 max=189109 max_rank=2 min=67692 min_rank=0 imbalance_ratio=1.442787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[99048, 98312, 124858, 165116, 165699, 245858, 116654, 33031] avg=131072.0000 max=245858 max_rank=5 min=33031 min_rank=7 imbalance_ratio=1.875748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[99048, 98312, 124858, 165116, 165699, 245858, 116654, 33031] avg=131072.0000 max=245858 max_rank=5 min=33031 min_rank=7 imbalance_ratio=1.875748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[67692, 130912, 189109, 112938, 163502, 100453, 161364, 122606] avg=131072.0000 max=189109 max_rank=2 min=67692 min_rank=0 imbalance_ratio=1.442787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[99048, 98312, 124858, 165116, 165699, 245858, 116654, 33031] avg=131072.0000 max=245858 max_rank=5 min=33031 min_rank=7 imbalance_ratio=1.875748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] avg=131072.0000 max=132383 max_rank=2 min=124416 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=56726 max_load_reduction_pct=0.299965 moved_to_replicas=235734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[99048, 98312, 124858, 165116, 165699, 245858, 116654, 33031] avg=131072.0000 max=245858 max_rank=5 min=33031 min_rank=7 imbalance_ratio=1.875748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] avg=131072.0000 max=132383 max_rank=2 min=124416 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=56726 max_load_reduction_pct=0.299965 moved_to_replicas=235734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] avg=131072.0000 max=132383 max_rank=2 min=124416 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=56726 max_load_reduction_pct=0.299965 moved_to_replicas=235734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] avg=131072.0000 max=132384 max_rank=7 min=124858 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=113474 max_load_reduction_pct=0.461543 moved_to_replicas=364234 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] avg=131072.0000 max=132384 max_rank=7 min=124858 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=113474 max_load_reduction_pct=0.461543 moved_to_replicas=364234 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[131574, 130912, 132383, 132382, 132383, 124416, 132383, 132143] avg=131072.0000 max=132383 max_rank=2 min=124416 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=56726 max_load_reduction_pct=0.299965 moved_to_replicas=235734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] avg=131072.0000 max=132384 max_rank=7 min=124858 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=113474 max_load_reduction_pct=0.461543 moved_to_replicas=364234 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] avg=131072.0000 max=132384 max_rank=7 min=124858 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=113474 max_load_reduction_pct=0.461543 moved_to_replicas=364234 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[99048, 98312, 124858, 165116, 165699, 245858, 116654, 33031] avg=131072.0000 max=245858 max_rank=5 min=33031 min_rank=7 imbalance_ratio=1.875748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[99048, 98312, 124858, 165116, 165699, 245858, 116654, 33031] avg=131072.0000 max=245858 max_rank=5 min=33031 min_rank=7 imbalance_ratio=1.875748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[99048, 98312, 124858, 165116, 165699, 245858, 116654, 33031] avg=131072.0000 max=245858 max_rank=5 min=33031 min_rank=7 imbalance_ratio=1.875748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[128952, 85729, 203602, 130946, 170588, 142788, 153169, 32802] avg=131072.0000 max=203602 max_rank=2 min=32802 min_rank=7 imbalance_ratio=1.553360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[99048, 98312, 124858, 165116, 165699, 245858, 116654, 33031] avg=131072.0000 max=245858 max_rank=5 min=33031 min_rank=7 imbalance_ratio=1.875748 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[128952, 85729, 203602, 130946, 170588, 142788, 153169, 32802] avg=131072.0000 max=203602 max_rank=2 min=32802 min_rank=7 imbalance_ratio=1.553360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[128952, 85729, 203602, 130946, 170588, 142788, 153169, 32802] avg=131072.0000 max=203602 max_rank=2 min=32802 min_rank=7 imbalance_ratio=1.553360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[128952, 85729, 203602, 130946, 170588, 142788, 153169, 32802] avg=131072.0000 max=203602 max_rank=2 min=32802 min_rank=7 imbalance_ratio=1.553360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] avg=131072.0000 max=132384 max_rank=7 min=124858 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=113474 max_load_reduction_pct=0.461543 moved_to_replicas=364234 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] avg=131072.0000 max=132384 max_rank=7 min=124858 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=113474 max_load_reduction_pct=0.461543 moved_to_replicas=364234 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] avg=131072.0000 max=132384 max_rank=7 min=124858 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=113474 max_load_reduction_pct=0.461543 moved_to_replicas=364234 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] avg=131072.0000 max=132385 max_rank=1 min=126762 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71217 max_load_reduction_pct=0.349785 moved_to_replicas=282818 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] avg=131072.0000 max=132385 max_rank=1 min=126762 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71217 max_load_reduction_pct=0.349785 moved_to_replicas=282818 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[131811, 131075, 124858, 132383, 132383, 132383, 131299, 132384] avg=131072.0000 max=132384 max_rank=7 min=124858 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=113474 max_load_reduction_pct=0.461543 moved_to_replicas=364234 active_clone_links=17 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] avg=131072.0000 max=132385 max_rank=1 min=126762 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71217 max_load_reduction_pct=0.349785 moved_to_replicas=282818 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] avg=131072.0000 max=132385 max_rank=1 min=126762 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71217 max_load_reduction_pct=0.349785 moved_to_replicas=282818 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[128952, 85729, 203602, 130946, 170588, 142788, 153169, 32802] avg=131072.0000 max=203602 max_rank=2 min=32802 min_rank=7 imbalance_ratio=1.553360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[128952, 85729, 203602, 130946, 170588, 142788, 153169, 32802] avg=131072.0000 max=203602 max_rank=2 min=32802 min_rank=7 imbalance_ratio=1.553360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[128952, 85729, 203602, 130946, 170588, 142788, 153169, 32802] avg=131072.0000 max=203602 max_rank=2 min=32802 min_rank=7 imbalance_ratio=1.553360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[101275, 96464, 207514, 98140, 78235, 163137, 66834, 236977] avg=131072.0000 max=236977 max_rank=7 min=66834 min_rank=6 imbalance_ratio=1.807991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[101275, 96464, 207514, 98140, 78235, 163137, 66834, 236977] avg=131072.0000 max=236977 max_rank=7 min=66834 min_rank=6 imbalance_ratio=1.807991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[128952, 85729, 203602, 130946, 170588, 142788, 153169, 32802] avg=131072.0000 max=203602 max_rank=2 min=32802 min_rank=7 imbalance_ratio=1.553360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[101275, 96464, 207514, 98140, 78235, 163137, 66834, 236977] avg=131072.0000 max=236977 max_rank=7 min=66834 min_rank=6 imbalance_ratio=1.807991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[101275, 96464, 207514, 98140, 78235, 163137, 66834, 236977] avg=131072.0000 max=236977 max_rank=7 min=66834 min_rank=6 imbalance_ratio=1.807991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] avg=131072.0000 max=132385 max_rank=1 min=126762 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71217 max_load_reduction_pct=0.349785 moved_to_replicas=282818 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] avg=131072.0000 max=132385 max_rank=1 min=126762 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71217 max_load_reduction_pct=0.349785 moved_to_replicas=282818 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] avg=131072.0000 max=132385 max_rank=1 min=126762 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71217 max_load_reduction_pct=0.349785 moved_to_replicas=282818 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] avg=131072.0000 max=132383 max_rank=0 min=125080 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104594 max_load_reduction_pct=0.441368 moved_to_replicas=421324 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] avg=131072.0000 max=132383 max_rank=0 min=125080 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104594 max_load_reduction_pct=0.441368 moved_to_replicas=421324 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[128952, 132385, 132381, 130946, 132383, 132383, 132384, 126762] avg=131072.0000 max=132385 max_rank=1 min=126762 min_rank=7 imbalance_ratio=1.010017 max_load_reduction=71217 max_load_reduction_pct=0.349785 moved_to_replicas=282818 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] avg=131072.0000 max=132383 max_rank=0 min=125080 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104594 max_load_reduction_pct=0.441368 moved_to_replicas=421324 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] avg=131072.0000 max=132383 max_rank=0 min=125080 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104594 max_load_reduction_pct=0.441368 moved_to_replicas=421324 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[101275, 96464, 207514, 98140, 78235, 163137, 66834, 236977] avg=131072.0000 max=236977 max_rank=7 min=66834 min_rank=6 imbalance_ratio=1.807991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[101275, 96464, 207514, 98140, 78235, 163137, 66834, 236977] avg=131072.0000 max=236977 max_rank=7 min=66834 min_rank=6 imbalance_ratio=1.807991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[101275, 96464, 207514, 98140, 78235, 163137, 66834, 236977] avg=131072.0000 max=236977 max_rank=7 min=66834 min_rank=6 imbalance_ratio=1.807991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[129929, 135183, 130593, 104831, 100739, 121189, 131743, 194369] avg=131072.0000 max=194369 max_rank=7 min=100739 min_rank=4 imbalance_ratio=1.482918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[129929, 135183, 130593, 104831, 100739, 121189, 131743, 194369] avg=131072.0000 max=194369 max_rank=7 min=100739 min_rank=4 imbalance_ratio=1.482918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[129929, 135183, 130593, 104831, 100739, 121189, 131743, 194369] avg=131072.0000 max=194369 max_rank=7 min=100739 min_rank=4 imbalance_ratio=1.482918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[101275, 96464, 207514, 98140, 78235, 163137, 66834, 236977] avg=131072.0000 max=236977 max_rank=7 min=66834 min_rank=6 imbalance_ratio=1.807991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[129929, 135183, 130593, 104831, 100739, 121189, 131743, 194369] avg=131072.0000 max=194369 max_rank=7 min=100739 min_rank=4 imbalance_ratio=1.482918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] avg=131072.0000 max=132383 max_rank=0 min=125080 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104594 max_load_reduction_pct=0.441368 moved_to_replicas=421324 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] avg=131072.0000 max=132383 max_rank=0 min=125080 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104594 max_load_reduction_pct=0.441368 moved_to_replicas=421324 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] avg=131072.0000 max=132383 max_rank=0 min=125080 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104594 max_load_reduction_pct=0.441368 moved_to_replicas=421324 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] avg=131072.0000 max=132383 max_rank=1 min=126779 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61986 max_load_reduction_pct=0.318909 moved_to_replicas=127346 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] avg=131072.0000 max=132383 max_rank=1 min=126779 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61986 max_load_reduction_pct=0.318909 moved_to_replicas=127346 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] avg=131072.0000 max=132383 max_rank=1 min=126779 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61986 max_load_reduction_pct=0.318909 moved_to_replicas=127346 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[132383, 129224, 132381, 125080, 132383, 132383, 132359, 132383] avg=131072.0000 max=132383 max_rank=0 min=125080 min_rank=3 imbalance_ratio=1.010002 max_load_reduction=104594 max_load_reduction_pct=0.441368 moved_to_replicas=421324 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] avg=131072.0000 max=132383 max_rank=1 min=126779 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61986 max_load_reduction_pct=0.318909 moved_to_replicas=127346 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[129929, 135183, 130593, 104831, 100739, 121189, 131743, 194369] avg=131072.0000 max=194369 max_rank=7 min=100739 min_rank=4 imbalance_ratio=1.482918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[129929, 135183, 130593, 104831, 100739, 121189, 131743, 194369] avg=131072.0000 max=194369 max_rank=7 min=100739 min_rank=4 imbalance_ratio=1.482918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[129929, 135183, 130593, 104831, 100739, 121189, 131743, 194369] avg=131072.0000 max=194369 max_rank=7 min=100739 min_rank=4 imbalance_ratio=1.482918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[154094, 97263, 125257, 98798, 16701, 228705, 98916, 228842] avg=131072.0000 max=228842 max_rank=7 min=16701 min_rank=4 imbalance_ratio=1.745926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[154094, 97263, 125257, 98798, 16701, 228705, 98916, 228842] avg=131072.0000 max=228842 max_rank=7 min=16701 min_rank=4 imbalance_ratio=1.745926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[154094, 97263, 125257, 98798, 16701, 228705, 98916, 228842] avg=131072.0000 max=228842 max_rank=7 min=16701 min_rank=4 imbalance_ratio=1.745926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[129929, 135183, 130593, 104831, 100739, 121189, 131743, 194369] avg=131072.0000 max=194369 max_rank=7 min=100739 min_rank=4 imbalance_ratio=1.482918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[154094, 97263, 125257, 98798, 16701, 228705, 98916, 228842] avg=131072.0000 max=228842 max_rank=7 min=16701 min_rank=4 imbalance_ratio=1.745926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] avg=131072.0000 max=132383 max_rank=1 min=126779 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61986 max_load_reduction_pct=0.318909 moved_to_replicas=127346 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] avg=131072.0000 max=132383 max_rank=1 min=126779 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61986 max_load_reduction_pct=0.318909 moved_to_replicas=127346 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] avg=131072.0000 max=132383 max_rank=1 min=126779 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61986 max_load_reduction_pct=0.318909 moved_to_replicas=127346 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] avg=131072.0000 max=132384 max_rank=4 min=127745 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96458 max_load_reduction_pct=0.421505 moved_to_replicas=428386 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] avg=131072.0000 max=132384 max_rank=4 min=127745 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96458 max_load_reduction_pct=0.421505 moved_to_replicas=428386 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] avg=131072.0000 max=132384 max_rank=4 min=127745 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96458 max_load_reduction_pct=0.421505 moved_to_replicas=428386 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] avg=131072.0000 max=132384 max_rank=4 min=127745 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96458 max_load_reduction_pct=0.421505 moved_to_replicas=428386 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[129929, 132383, 130593, 132383, 132383, 126779, 131743, 132383] avg=131072.0000 max=132383 max_rank=1 min=126779 min_rank=5 imbalance_ratio=1.010002 max_load_reduction=61986 max_load_reduction_pct=0.318909 moved_to_replicas=127346 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[154094, 97263, 125257, 98798, 16701, 228705, 98916, 228842] avg=131072.0000 max=228842 max_rank=7 min=16701 min_rank=4 imbalance_ratio=1.745926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[154094, 97263, 125257, 98798, 16701, 228705, 98916, 228842] avg=131072.0000 max=228842 max_rank=7 min=16701 min_rank=4 imbalance_ratio=1.745926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[154094, 97263, 125257, 98798, 16701, 228705, 98916, 228842] avg=131072.0000 max=228842 max_rank=7 min=16701 min_rank=4 imbalance_ratio=1.745926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[33725, 196631, 56485, 172566, 63643, 198072, 98520, 228934] avg=131072.0000 max=228934 max_rank=7 min=33725 min_rank=0 imbalance_ratio=1.746628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[33725, 196631, 56485, 172566, 63643, 198072, 98520, 228934] avg=131072.0000 max=228934 max_rank=7 min=33725 min_rank=0 imbalance_ratio=1.746628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[33725, 196631, 56485, 172566, 63643, 198072, 98520, 228934] avg=131072.0000 max=228934 max_rank=7 min=33725 min_rank=0 imbalance_ratio=1.746628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[33725, 196631, 56485, 172566, 63643, 198072, 98520, 228934] avg=131072.0000 max=228934 max_rank=7 min=33725 min_rank=0 imbalance_ratio=1.746628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[154094, 97263, 125257, 98798, 16701, 228705, 98916, 228842] avg=131072.0000 max=228842 max_rank=7 min=16701 min_rank=4 imbalance_ratio=1.745926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] avg=131072.0000 max=132384 max_rank=4 min=127745 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96458 max_load_reduction_pct=0.421505 moved_to_replicas=428386 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] avg=131072.0000 max=132384 max_rank=4 min=127745 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96458 max_load_reduction_pct=0.421505 moved_to_replicas=428386 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] avg=131072.0000 max=132384 max_rank=4 min=127745 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96458 max_load_reduction_pct=0.421505 moved_to_replicas=428386 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] avg=131072.0000 max=132383 max_rank=1 min=127599 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96551 max_load_reduction_pct=0.421742 moved_to_replicas=533926 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] avg=131072.0000 max=132383 max_rank=1 min=127599 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96551 max_load_reduction_pct=0.421742 moved_to_replicas=533926 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] avg=131072.0000 max=132383 max_rank=1 min=127599 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96551 max_load_reduction_pct=0.421742 moved_to_replicas=533926 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] avg=131072.0000 max=132383 max_rank=1 min=127599 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96551 max_load_reduction_pct=0.421742 moved_to_replicas=533926 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[132382, 130026, 127745, 131560, 132384, 132383, 129713, 132383] avg=131072.0000 max=132384 max_rank=4 min=127745 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96458 max_load_reduction_pct=0.421505 moved_to_replicas=428386 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[33725, 196631, 56485, 172566, 63643, 198072, 98520, 228934] avg=131072.0000 max=228934 max_rank=7 min=33725 min_rank=0 imbalance_ratio=1.746628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[33725, 196631, 56485, 172566, 63643, 198072, 98520, 228934] avg=131072.0000 max=228934 max_rank=7 min=33725 min_rank=0 imbalance_ratio=1.746628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[33725, 196631, 56485, 172566, 63643, 198072, 98520, 228934] avg=131072.0000 max=228934 max_rank=7 min=33725 min_rank=0 imbalance_ratio=1.746628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[261758, 118728, 103744, 159452, 97896, 141985, 83009, 82004] avg=131072.0000 max=261758 max_rank=0 min=82004 min_rank=7 imbalance_ratio=1.997055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[261758, 118728, 103744, 159452, 97896, 141985, 83009, 82004] avg=131072.0000 max=261758 max_rank=0 min=82004 min_rank=7 imbalance_ratio=1.997055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[261758, 118728, 103744, 159452, 97896, 141985, 83009, 82004] avg=131072.0000 max=261758 max_rank=0 min=82004 min_rank=7 imbalance_ratio=1.997055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[261758, 118728, 103744, 159452, 97896, 141985, 83009, 82004] avg=131072.0000 max=261758 max_rank=0 min=82004 min_rank=7 imbalance_ratio=1.997055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[33725, 196631, 56485, 172566, 63643, 198072, 98520, 228934] avg=131072.0000 max=228934 max_rank=7 min=33725 min_rank=0 imbalance_ratio=1.746628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] avg=131072.0000 max=132383 max_rank=1 min=127599 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96551 max_load_reduction_pct=0.421742 moved_to_replicas=533926 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] avg=131072.0000 max=132383 max_rank=1 min=127599 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96551 max_load_reduction_pct=0.421742 moved_to_replicas=533926 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] avg=131072.0000 max=132383 max_rank=1 min=127599 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96551 max_load_reduction_pct=0.421742 moved_to_replicas=533926 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] avg=131072.0000 max=132384 max_rank=3 min=127672 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=129374 max_load_reduction_pct=0.494250 moved_to_replicas=330585 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] avg=131072.0000 max=132384 max_rank=3 min=127672 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=129374 max_load_reduction_pct=0.494250 moved_to_replicas=330585 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] avg=131072.0000 max=132384 max_rank=3 min=127672 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=129374 max_load_reduction_pct=0.494250 moved_to_replicas=330585 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] avg=131072.0000 max=132384 max_rank=3 min=127672 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=129374 max_load_reduction_pct=0.494250 moved_to_replicas=330585 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[132014, 132383, 129426, 132383, 127599, 132383, 130005, 132383] avg=131072.0000 max=132383 max_rank=1 min=127599 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=96551 max_load_reduction_pct=0.421742 moved_to_replicas=533926 active_clone_links=20 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[261758, 118728, 103744, 159452, 97896, 141985, 83009, 82004] avg=131072.0000 max=261758 max_rank=0 min=82004 min_rank=7 imbalance_ratio=1.997055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[261758, 118728, 103744, 159452, 97896, 141985, 83009, 82004] avg=131072.0000 max=261758 max_rank=0 min=82004 min_rank=7 imbalance_ratio=1.997055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[261758, 118728, 103744, 159452, 97896, 141985, 83009, 82004] avg=131072.0000 max=261758 max_rank=0 min=82004 min_rank=7 imbalance_ratio=1.997055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[125157, 229269, 32881, 130830, 130331, 169965, 164200, 65943] avg=131072.0000 max=229269 max_rank=1 min=32881 min_rank=2 imbalance_ratio=1.749184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[125157, 229269, 32881, 130830, 130331, 169965, 164200, 65943] avg=131072.0000 max=229269 max_rank=1 min=32881 min_rank=2 imbalance_ratio=1.749184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[125157, 229269, 32881, 130830, 130331, 169965, 164200, 65943] avg=131072.0000 max=229269 max_rank=1 min=32881 min_rank=2 imbalance_ratio=1.749184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[125157, 229269, 32881, 130830, 130331, 169965, 164200, 65943] avg=131072.0000 max=229269 max_rank=1 min=32881 min_rank=2 imbalance_ratio=1.749184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[261758, 118728, 103744, 159452, 97896, 141985, 83009, 82004] avg=131072.0000 max=261758 max_rank=0 min=82004 min_rank=7 imbalance_ratio=1.997055 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] avg=131072.0000 max=132384 max_rank=3 min=127672 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=129374 max_load_reduction_pct=0.494250 moved_to_replicas=330585 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] avg=131072.0000 max=132384 max_rank=3 min=127672 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=129374 max_load_reduction_pct=0.494250 moved_to_replicas=330585 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] avg=131072.0000 max=132384 max_rank=3 min=127672 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=129374 max_load_reduction_pct=0.494250 moved_to_replicas=330585 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] avg=131072.0000 max=132385 max_rank=2 min=127814 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=96884 max_load_reduction_pct=0.422578 moved_to_replicas=330980 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] avg=131072.0000 max=132385 max_rank=2 min=127814 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=96884 max_load_reduction_pct=0.422578 moved_to_replicas=330980 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] avg=131072.0000 max=132385 max_rank=2 min=127814 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=96884 max_load_reduction_pct=0.422578 moved_to_replicas=330980 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] avg=131072.0000 max=132385 max_rank=2 min=127814 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=96884 max_load_reduction_pct=0.422578 moved_to_replicas=330980 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[132383, 128331, 132383, 132384, 130657, 132382, 132384, 127672] avg=131072.0000 max=132384 max_rank=3 min=127672 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=129374 max_load_reduction_pct=0.494250 moved_to_replicas=330585 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[125157, 229269, 32881, 130830, 130331, 169965, 164200, 65943] avg=131072.0000 max=229269 max_rank=1 min=32881 min_rank=2 imbalance_ratio=1.749184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[125157, 229269, 32881, 130830, 130331, 169965, 164200, 65943] avg=131072.0000 max=229269 max_rank=1 min=32881 min_rank=2 imbalance_ratio=1.749184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[125157, 229269, 32881, 130830, 130331, 169965, 164200, 65943] avg=131072.0000 max=229269 max_rank=1 min=32881 min_rank=2 imbalance_ratio=1.749184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[76402, 195560, 431, 226547, 84358, 224500, 110274, 130504] avg=131072.0000 max=226547 max_rank=3 min=431 min_rank=2 imbalance_ratio=1.728416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[76402, 195560, 431, 226547, 84358, 224500, 110274, 130504] avg=131072.0000 max=226547 max_rank=3 min=431 min_rank=2 imbalance_ratio=1.728416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[76402, 195560, 431, 226547, 84358, 224500, 110274, 130504] avg=131072.0000 max=226547 max_rank=3 min=431 min_rank=2 imbalance_ratio=1.728416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] avg=131072.0000 max=132385 max_rank=2 min=127814 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=96884 max_load_reduction_pct=0.422578 moved_to_replicas=330980 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[76402, 195560, 431, 226547, 84358, 224500, 110274, 130504] avg=131072.0000 max=226547 max_rank=3 min=431 min_rank=2 imbalance_ratio=1.728416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[125157, 229269, 32881, 130830, 130331, 169965, 164200, 65943] avg=131072.0000 max=229269 max_rank=1 min=32881 min_rank=2 imbalance_ratio=1.749184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] avg=131072.0000 max=132385 max_rank=2 min=127814 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=96884 max_load_reduction_pct=0.422578 moved_to_replicas=330980 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] avg=131072.0000 max=132385 max_rank=2 min=127814 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=96884 max_load_reduction_pct=0.422578 moved_to_replicas=330980 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] avg=131072.0000 max=132383 max_rank=1 min=127775 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94164 max_load_reduction_pct=0.415649 moved_to_replicas=497164 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] avg=131072.0000 max=132383 max_rank=1 min=127775 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94164 max_load_reduction_pct=0.415649 moved_to_replicas=497164 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] avg=131072.0000 max=132383 max_rank=1 min=127775 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94164 max_load_reduction_pct=0.415649 moved_to_replicas=497164 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] avg=131072.0000 max=132383 max_rank=1 min=127775 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94164 max_load_reduction_pct=0.415649 moved_to_replicas=497164 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[127814, 132383, 132385, 130830, 130331, 132381, 132383, 130069] avg=131072.0000 max=132385 max_rank=2 min=127814 min_rank=0 imbalance_ratio=1.010017 max_load_reduction=96884 max_load_reduction_pct=0.422578 moved_to_replicas=330980 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[76402, 195560, 431, 226547, 84358, 224500, 110274, 130504] avg=131072.0000 max=226547 max_rank=3 min=431 min_rank=2 imbalance_ratio=1.728416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[76402, 195560, 431, 226547, 84358, 224500, 110274, 130504] avg=131072.0000 max=226547 max_rank=3 min=431 min_rank=2 imbalance_ratio=1.728416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[99855, 131035, 70605, 98206, 154312, 151586, 229256, 113721] avg=131072.0000 max=229256 max_rank=6 min=70605 min_rank=2 imbalance_ratio=1.749084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[99855, 131035, 70605, 98206, 154312, 151586, 229256, 113721] avg=131072.0000 max=229256 max_rank=6 min=70605 min_rank=2 imbalance_ratio=1.749084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[76402, 195560, 431, 226547, 84358, 224500, 110274, 130504] avg=131072.0000 max=226547 max_rank=3 min=431 min_rank=2 imbalance_ratio=1.728416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[99855, 131035, 70605, 98206, 154312, 151586, 229256, 113721] avg=131072.0000 max=229256 max_rank=6 min=70605 min_rank=2 imbalance_ratio=1.749084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[99855, 131035, 70605, 98206, 154312, 151586, 229256, 113721] avg=131072.0000 max=229256 max_rank=6 min=70605 min_rank=2 imbalance_ratio=1.749084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] avg=131072.0000 max=132383 max_rank=1 min=127775 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94164 max_load_reduction_pct=0.415649 moved_to_replicas=497164 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[76402, 195560, 431, 226547, 84358, 224500, 110274, 130504] avg=131072.0000 max=226547 max_rank=3 min=431 min_rank=2 imbalance_ratio=1.728416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] avg=131072.0000 max=132383 max_rank=1 min=127775 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94164 max_load_reduction_pct=0.415649 moved_to_replicas=497164 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] avg=131072.0000 max=132384 max_rank=5 min=125839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96872 max_load_reduction_pct=0.422549 moved_to_replicas=275935 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] avg=131072.0000 max=132384 max_rank=5 min=125839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96872 max_load_reduction_pct=0.422549 moved_to_replicas=275935 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] avg=131072.0000 max=132383 max_rank=1 min=127775 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94164 max_load_reduction_pct=0.415649 moved_to_replicas=497164 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] avg=131072.0000 max=132384 max_rank=5 min=125839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96872 max_load_reduction_pct=0.422549 moved_to_replicas=275935 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] avg=131072.0000 max=132384 max_rank=5 min=125839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96872 max_load_reduction_pct=0.422549 moved_to_replicas=275935 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[132382, 132383, 128383, 132383, 127775, 132383, 132383, 130504] avg=131072.0000 max=132383 max_rank=1 min=127775 min_rank=4 imbalance_ratio=1.010002 max_load_reduction=94164 max_load_reduction_pct=0.415649 moved_to_replicas=497164 active_clone_links=21 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[99855, 131035, 70605, 98206, 154312, 151586, 229256, 113721] avg=131072.0000 max=229256 max_rank=6 min=70605 min_rank=2 imbalance_ratio=1.749084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[99855, 131035, 70605, 98206, 154312, 151586, 229256, 113721] avg=131072.0000 max=229256 max_rank=6 min=70605 min_rank=2 imbalance_ratio=1.749084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[32978, 181909, 59647, 131272, 68528, 198293, 196542, 179407] avg=131072.0000 max=198293 max_rank=5 min=32978 min_rank=0 imbalance_ratio=1.512856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[32978, 181909, 59647, 131272, 68528, 198293, 196542, 179407] avg=131072.0000 max=198293 max_rank=5 min=32978 min_rank=0 imbalance_ratio=1.512856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[99855, 131035, 70605, 98206, 154312, 151586, 229256, 113721] avg=131072.0000 max=229256 max_rank=6 min=70605 min_rank=2 imbalance_ratio=1.749084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[32978, 181909, 59647, 131272, 68528, 198293, 196542, 179407] avg=131072.0000 max=198293 max_rank=5 min=32978 min_rank=0 imbalance_ratio=1.512856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[32978, 181909, 59647, 131272, 68528, 198293, 196542, 179407] avg=131072.0000 max=198293 max_rank=5 min=32978 min_rank=0 imbalance_ratio=1.512856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[99855, 131035, 70605, 98206, 154312, 151586, 229256, 113721] avg=131072.0000 max=229256 max_rank=6 min=70605 min_rank=2 imbalance_ratio=1.749084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] avg=131072.0000 max=132384 max_rank=5 min=125839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96872 max_load_reduction_pct=0.422549 moved_to_replicas=275935 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] avg=131072.0000 max=132384 max_rank=5 min=125839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96872 max_load_reduction_pct=0.422549 moved_to_replicas=275935 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=4 min=125130 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=65910 max_load_reduction_pct=0.332387 moved_to_replicas=453283 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=4 min=125130 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=65910 max_load_reduction_pct=0.332387 moved_to_replicas=453283 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] avg=131072.0000 max=132384 max_rank=5 min=125839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96872 max_load_reduction_pct=0.422549 moved_to_replicas=275935 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=4 min=125130 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=65910 max_load_reduction_pct=0.332387 moved_to_replicas=453283 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=4 min=125130 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=65910 max_load_reduction_pct=0.332387 moved_to_replicas=453283 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[131202, 131035, 125839, 130969, 132382, 132384, 132383, 132382] avg=131072.0000 max=132384 max_rank=5 min=125839 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=96872 max_load_reduction_pct=0.422549 moved_to_replicas=275935 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[32978, 181909, 59647, 131272, 68528, 198293, 196542, 179407] avg=131072.0000 max=198293 max_rank=5 min=32978 min_rank=0 imbalance_ratio=1.512856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[32978, 181909, 59647, 131272, 68528, 198293, 196542, 179407] avg=131072.0000 max=198293 max_rank=5 min=32978 min_rank=0 imbalance_ratio=1.512856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[32978, 181909, 59647, 131272, 68528, 198293, 196542, 179407] avg=131072.0000 max=198293 max_rank=5 min=32978 min_rank=0 imbalance_ratio=1.512856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[32978, 181909, 59647, 131272, 68528, 198293, 196542, 179407] avg=131072.0000 max=198293 max_rank=5 min=32978 min_rank=0 imbalance_ratio=1.512856 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=4 min=125130 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=65910 max_load_reduction_pct=0.332387 moved_to_replicas=453283 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=4 min=125130 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=65910 max_load_reduction_pct=0.332387 moved_to_replicas=453283 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=4 min=125130 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=65910 max_load_reduction_pct=0.332387 moved_to_replicas=453283 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[130261, 132381, 125130, 131272, 132383, 132383, 132383, 132383] avg=131072.0000 max=132383 max_rank=4 min=125130 min_rank=2 imbalance_ratio=1.010002 max_load_reduction=65910 max_load_reduction_pct=0.332387 moved_to_replicas=453283 active_clone_links=18 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=4187124 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014144 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=4160146 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1041792 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=4199360 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1006080 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=4206472 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=997760 peak_demand=132384 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=4201241 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1001344 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=4204271 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1003904 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=4198907 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1001600 peak_demand=132385 required_capacity=1.010017 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=4196911 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1004800 peak_demand=132384 required_capacity=1.010010 + [2026-08-30 05:17:58] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 584.1 | throughput per GPU (TFLOP/s/GPU): 570.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.690156E+00 | loss scale: 1.0 | grad norm: 4.917 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 137984 | min safe capacity: 1.0527 | +[after training is done] datetime: 2026-08-30 05:17:58 +[rank0]:[W830 05:17:59.645581682 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 05:18:00.973229677 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 05:18:00.995858338 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 05:18:00.995869959 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 05:18:00.997565996 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 05:18:00.998334951 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 05:18:01.504636995 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 05:18:01.639568200 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:18:03.349000 505547 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:18:03.349000 505547 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 505639 closing signal SIGTERM +W0830 05:18:03.350000 505547 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 505640 closing signal SIGTERM +W0830 05:18:03.351000 505547 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 505641 closing signal SIGTERM +W0830 05:18:03.351000 505547 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 505642 closing signal SIGTERM +W0830 05:18:03.351000 505547 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 505643 closing signal SIGTERM +W0830 05:18:03.352000 505547 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 505644 closing signal SIGTERM +W0830 05:18:03.352000 505547 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 505645 closing signal SIGTERM +W0830 05:18:03.353000 505547 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 505646 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 505547 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..6f7d8fb3c368bd38f18cbb28f08dc0e91f4e2a5d --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 35 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 947 +1, 955 +2, 955 +3, 969 +4, 957 +5, 973 +6, 977 +7, 969 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 2411 +1, 2441 +2, 2413 +3, 2429 +4, 2445 +5, 2421 +6, 2443 +7, 2427 +0, 10421 +1, 10853 +2, 10933 +3, 10933 +4, 10653 +5, 10421 +6, 10853 +7, 10613 +0, 42851 +1, 43299 +2, 43299 +3, 43299 +4, 42851 +5, 43299 +6, 43299 +7, 43299 +0, 43745 +1, 44193 +2, 44193 +3, 44193 +4, 43745 +5, 44193 +6, 44193 +7, 44193 +0, 44429 +1, 44877 +2, 44877 +3, 44877 +4, 44429 +5, 44877 +6, 44877 +7, 44877 +0, 45709 +1, 46157 +2, 46157 +3, 46157 +4, 45709 +5, 46157 +6, 46157 +7, 46157 +0, 48843 +1, 49291 +2, 49291 +3, 49291 +4, 48843 +5, 49291 +6, 49291 +7, 49291 +0, 65585 +1, 65619 +2, 65619 +3, 65617 +4, 65611 +5, 65613 +6, 65615 +7, 65613 +0, 82765 +1, 82813 +2, 82813 +3, 82813 +4, 82785 +5, 82813 +6, 82813 +7, 82813 +0, 84045 +1, 84093 +2, 84093 +3, 84093 +4, 84065 +5, 84093 +6, 84093 +7, 84093 +0, 84045 +1, 84093 +2, 84093 +3, 84093 +4, 84065 +5, 84093 +6, 84093 +7, 84093 +0, 95279 +1, 95407 +2, 95377 +3, 95417 +4, 95399 +5, 95497 +6, 95457 +7, 95477 +0, 96129 +1, 96143 +2, 96171 +3, 96169 +4, 96139 +5, 96165 +6, 96173 +7, 96157 +0, 97173 +1, 97195 +2, 97211 +3, 97187 +4, 97233 +5, 97223 +6, 97179 +7, 97205 +0, 120387 +1, 120475 +2, 120475 +3, 120475 +4, 120487 +5, 120475 +6, 120475 +7, 120475 +0, 120387 +1, 120475 +2, 120475 +3, 120475 +4, 120487 +5, 120475 +6, 120475 +7, 120475 +0, 120387 +1, 120475 +2, 120475 +3, 120475 +4, 120487 +5, 120475 +6, 120475 +7, 120475 +0, 119441 +1, 119529 +2, 119529 +3, 119529 +4, 119541 +5, 119529 +6, 119529 +7, 119529 +0, 115193 +1, 115281 +2, 115281 +3, 115281 +4, 115293 +5, 115281 +6, 115669 +7, 115281 +0, 113125 +1, 113213 +2, 113601 +3, 113213 +4, 113613 +5, 113601 +6, 113601 +7, 113601 +0, 3 +1, 4 +2, 113601 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..2772d6fa3c134827da4e460e05263a7f9d8fe14c --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G4 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=896 +num_experts=128 +topk=32 +experts_per_rank=16 +expert_params=11010048 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=131072 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6ddb819bdff31cf00354bf1a1a501f71db50f3bb --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log @@ -0,0 +1,9351 @@ +[05:19:24] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G4 / ultraep (slots/rank=16 policy=matched) + E=128 K=32 H'=896 m=16 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:19:25.731000 512593 torch/distributed/run.py:766] +W0830 05:19:25.731000 512593 torch/distributed/run.py:766] ***************************************** +W0830 05:19:25.731000 512593 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:19:25.731000 512593 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 896 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 16 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 32 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank [Gloo] Rank 0[Gloo] Rank [Gloo] Rank 31 is connected to 2 is connected to is connected to 7 is connected to 77 peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 77 +7 + + +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 45 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank4]:[W830 05:19:43.988365325 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:19:43.989690262 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:19:43.989704461 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:19:43.989753934 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:19:43.989871539 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:19:43.989881315 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:19:43.989968377 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:19:43.206641121 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.965 seconds +time to initialize megatron (seconds): 7.144 +[after megatron is initialized] datetime: 2026-08-30 05:19:49 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6728847360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42479616 elements, 42479616 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (42475520 elements, 42475520 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_proj.weight +Params for bucket 4 (42475520 elements, 42475520 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 5 (42475520 elements, 42475520 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.pre_mlp_layernorm.weight +Params for bucket 6 (42475520 elements, 42475520 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight +Params for bucket 7 (42475520 elements, 42475520 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 8 (42475520 elements, 42475520 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 9 (42475520 elements, 42475520 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 10 (42475520 elements, 42475520 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 11 (42475520 elements, 42475520 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight +Params for bucket 12 (42475520 elements, 42475520 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 13 (42475520 elements, 42475520 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight +Params for bucket 14 (42475520 elements, 42475520 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 15 (42475520 elements, 42475520 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight +Params for bucket 16 (42475520 elements, 42475520 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 17 (42475520 elements, 42475520 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 42 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 +Params for bucket 43 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 46 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 47 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 51 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 +Params for bucket 52 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 55 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 +Params for bucket 56 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 59 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 +Params for bucket 60 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 +Params for bucket 63 (40370176 elements, 40370176 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 +Params for bucket 64 (40370176 elements, 40370176 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 +Params for bucket 66 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 67 (29360128 elements, 29360128 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:19:54 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:19:54 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (4242.03, 4256.99) + train/valid/test-data-iterators-setup ..........: (76.66, 91.09) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:19:54 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=163840 expected_recv=261452 expected_overflow=0 exact_recv_per_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=163840 expected_recv=261452 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262205 expected_overflow=0 exact_recv_per_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=163840 expected_recv=261450 expected_overflow=0 exact_recv_per_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262428 expected_overflow=0 exact_recv_per_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262715 expected_overflow=0 exact_recv_per_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262082 expected_overflow=0 exact_recv_per_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262020 expected_overflow=0 exact_recv_per_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=163840 expected_recv=261822 expected_overflow=0 exact_recv_per_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=163840 expected_recv=261822 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=163840 expected_recv=261450 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262205 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262428 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262715 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262082 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262020 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=163840 expected_recv=261776 expected_overflow=0 exact_recv_per_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=163840 expected_recv=261776 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=163840 expected_recv=262263 expected_overflow=0 exact_recv_per_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=163840 expected_recv=262789 expected_overflow=0 exact_recv_per_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=163840 expected_recv=262875 expected_overflow=0 exact_recv_per_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=163840 expected_recv=261656 expected_overflow=0 exact_recv_per_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=163840 expected_recv=261672 expected_overflow=0 exact_recv_per_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=163840 expected_recv=262263 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=163840 expected_recv=262789 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=163840 expected_recv=262032 expected_overflow=0 exact_recv_per_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=163840 expected_recv=262875 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=163840 expected_recv=261656 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=163840 expected_recv=261672 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=163840 expected_recv=261891 expected_overflow=0 exact_recv_per_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=163840 expected_recv=262032 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=163840 expected_recv=261891 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=163840 expected_recv=263054 expected_overflow=0 exact_recv_per_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=163840 expected_recv=261973 expected_overflow=0 exact_recv_per_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=163840 expected_recv=261782 expected_overflow=0 exact_recv_per_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=163840 expected_recv=262028 expected_overflow=0 exact_recv_per_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=163840 expected_recv=263054 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=163840 expected_recv=261970 expected_overflow=0 exact_recv_per_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=163840 expected_recv=261973 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=163840 expected_recv=261782 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=163840 expected_recv=262028 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=163840 expected_recv=261466 expected_overflow=0 exact_recv_per_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=163840 expected_recv=261970 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=163840 expected_recv=262496 expected_overflow=0 exact_recv_per_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=163840 expected_recv=261466 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=163840 expected_recv=262496 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262800 expected_overflow=0 exact_recv_per_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=163840 expected_recv=261933 expected_overflow=0 exact_recv_per_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=163840 expected_recv=261776 expected_overflow=0 exact_recv_per_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=163840 expected_recv=262119 expected_overflow=0 exact_recv_per_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=163840 expected_recv=262506 expected_overflow=0 exact_recv_per_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=163840 expected_recv=261760 expected_overflow=0 exact_recv_per_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=163840 expected_recv=261933 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=163840 expected_recv=261776 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=163840 expected_recv=261989 expected_overflow=0 exact_recv_per_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=163840 expected_recv=262119 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=163840 expected_recv=262506 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=163840 expected_recv=262469 expected_overflow=0 exact_recv_per_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=163840 expected_recv=261760 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=163840 expected_recv=262800 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=163840 expected_recv=261989 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=163840 expected_recv=262469 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262316 expected_overflow=0 exact_recv_per_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262340 expected_overflow=0 exact_recv_per_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=163840 expected_recv=261908 expected_overflow=0 exact_recv_per_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262283 expected_overflow=0 exact_recv_per_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=163840 expected_recv=261804 expected_overflow=0 exact_recv_per_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262379 expected_overflow=0 exact_recv_per_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262316 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262340 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=163840 expected_recv=261908 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262283 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=163840 expected_recv=261804 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262379 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262275 expected_overflow=0 exact_recv_per_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262275 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=163840 expected_recv=262043 expected_overflow=0 exact_recv_per_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=163840 expected_recv=262043 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=163840 expected_recv=262224 expected_overflow=0 exact_recv_per_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=163840 expected_recv=261448 expected_overflow=0 exact_recv_per_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=163840 expected_recv=262042 expected_overflow=0 exact_recv_per_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=163840 expected_recv=261342 expected_overflow=0 exact_recv_per_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=163840 expected_recv=261878 expected_overflow=0 exact_recv_per_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=163840 expected_recv=262224 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=163840 expected_recv=261448 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=163840 expected_recv=262042 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=163840 expected_recv=261342 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=163840 expected_recv=262453 expected_overflow=0 exact_recv_per_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=163840 expected_recv=261878 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=163840 expected_recv=262071 expected_overflow=0 exact_recv_per_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=163840 expected_recv=262453 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=163840 expected_recv=262071 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=163840 expected_recv=262073 expected_overflow=0 exact_recv_per_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=163840 expected_recv=262073 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=163840 expected_recv=262591 expected_overflow=0 exact_recv_per_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=163840 expected_recv=262105 expected_overflow=0 exact_recv_per_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=163840 expected_recv=262591 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=163840 expected_recv=261610 expected_overflow=0 exact_recv_per_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=163840 expected_recv=262269 expected_overflow=0 exact_recv_per_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=163840 expected_recv=261881 expected_overflow=0 exact_recv_per_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=163840 expected_recv=261865 expected_overflow=0 exact_recv_per_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=163840 expected_recv=261988 expected_overflow=0 exact_recv_per_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=163840 expected_recv=262105 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=163840 expected_recv=261610 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=163840 expected_recv=262269 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=163840 expected_recv=261881 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=163840 expected_recv=261865 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=163840 expected_recv=261988 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=163840 expected_recv=262698 expected_overflow=0 exact_recv_per_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=163840 expected_recv=262698 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262846 expected_overflow=0 exact_recv_per_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262002 expected_overflow=0 exact_recv_per_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=163840 expected_recv=261988 expected_overflow=0 exact_recv_per_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262297 expected_overflow=0 exact_recv_per_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262095 expected_overflow=0 exact_recv_per_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=163840 expected_recv=260946 expected_overflow=0 exact_recv_per_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=163840 expected_recv=263029 expected_overflow=0 exact_recv_per_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262846 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262002 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=163840 expected_recv=261988 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262297 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262095 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=163840 expected_recv=260946 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=163840 expected_recv=263029 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262185 expected_overflow=0 exact_recv_per_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=163840 expected_recv=262185 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=163840 expected_recv=261651 expected_overflow=0 exact_recv_per_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262116 expected_overflow=0 exact_recv_per_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262259 expected_overflow=0 exact_recv_per_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262331 expected_overflow=0 exact_recv_per_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=163840 expected_recv=261651 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262307 expected_overflow=0 exact_recv_per_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262494 expected_overflow=0 exact_recv_per_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262403 expected_overflow=0 exact_recv_per_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262116 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262259 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262331 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262307 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262403 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=163840 expected_recv=263047 expected_overflow=0 exact_recv_per_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262494 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=163840 expected_recv=263047 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=163840 expected_recv=262567 expected_overflow=0 exact_recv_per_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=163840 expected_recv=262567 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=163840 expected_recv=261405 expected_overflow=0 exact_recv_per_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=163840 expected_recv=262715 expected_overflow=0 exact_recv_per_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=163840 expected_recv=262979 expected_overflow=0 exact_recv_per_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=163840 expected_recv=261639 expected_overflow=0 exact_recv_per_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=163840 expected_recv=261961 expected_overflow=0 exact_recv_per_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=163840 expected_recv=261405 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=163840 expected_recv=262397 expected_overflow=0 exact_recv_per_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=163840 expected_recv=262715 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=163840 expected_recv=262979 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=163840 expected_recv=262649 expected_overflow=0 exact_recv_per_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=163840 expected_recv=261639 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=163840 expected_recv=261961 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=163840 expected_recv=262397 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=163840 expected_recv=262649 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262404 expected_overflow=0 exact_recv_per_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262404 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262222 expected_overflow=0 exact_recv_per_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262180 expected_overflow=0 exact_recv_per_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=163840 expected_recv=261800 expected_overflow=0 exact_recv_per_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262385 expected_overflow=0 exact_recv_per_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=163840 expected_recv=261569 expected_overflow=0 exact_recv_per_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262214 expected_overflow=0 exact_recv_per_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262222 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262595 expected_overflow=0 exact_recv_per_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262180 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=163840 expected_recv=261800 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262385 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=163840 expected_recv=261569 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=163840 expected_recv=262214 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=163840 expected_recv=262595 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=163840 expected_recv=261654 expected_overflow=0 exact_recv_per_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=163840 expected_recv=261654 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=163840 expected_recv=262032 expected_overflow=0 exact_recv_per_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=163840 expected_recv=262921 expected_overflow=0 exact_recv_per_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=163840 expected_recv=262048 expected_overflow=0 exact_recv_per_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=163840 expected_recv=262377 expected_overflow=0 exact_recv_per_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=163840 expected_recv=261911 expected_overflow=0 exact_recv_per_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=163840 expected_recv=261327 expected_overflow=0 exact_recv_per_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=163840 expected_recv=262032 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=163840 expected_recv=262921 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262312 expected_overflow=0 exact_recv_per_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=163840 expected_recv=262048 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=163840 expected_recv=262377 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=163840 expected_recv=261911 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=163840 expected_recv=261327 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=163840 expected_recv=262312 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=163840 expected_recv=262771 expected_overflow=0 exact_recv_per_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=163840 expected_recv=262771 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=163840 expected_recv=262691 expected_overflow=0 exact_recv_per_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=163840 expected_recv=262221 expected_overflow=0 exact_recv_per_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=163840 expected_recv=261503 expected_overflow=0 exact_recv_per_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=163840 expected_recv=262691 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=163840 expected_recv=261687 expected_overflow=0 exact_recv_per_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=163840 expected_recv=262303 expected_overflow=0 exact_recv_per_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=163840 expected_recv=262221 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=163840 expected_recv=261503 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=163840 expected_recv=261165 expected_overflow=0 exact_recv_per_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=163840 expected_recv=262480 expected_overflow=0 exact_recv_per_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=163840 expected_recv=261687 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=163840 expected_recv=262303 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=163840 expected_recv=261165 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=163840 expected_recv=262146 expected_overflow=0 exact_recv_per_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=163840 expected_recv=262480 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=163840 expected_recv=262146 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=163840 expected_recv=262529 expected_overflow=0 exact_recv_per_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=163840 expected_recv=262738 expected_overflow=0 exact_recv_per_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=163840 expected_recv=261942 expected_overflow=0 exact_recv_per_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=163840 expected_recv=262529 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=163840 expected_recv=262738 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=163840 expected_recv=261952 expected_overflow=0 exact_recv_per_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=163840 expected_recv=262322 expected_overflow=0 exact_recv_per_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=163840 expected_recv=261942 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=163840 expected_recv=261847 expected_overflow=0 exact_recv_per_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=163840 expected_recv=261952 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=163840 expected_recv=261966 expected_overflow=0 exact_recv_per_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=163840 expected_recv=262322 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=163840 expected_recv=261967 expected_overflow=0 exact_recv_per_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=163840 expected_recv=261847 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=163840 expected_recv=261966 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=163840 expected_recv=261967 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=163840 expected_recv=262105 expected_overflow=0 exact_recv_per_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=163840 expected_recv=262105 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=163840 expected_recv=263245 expected_overflow=0 exact_recv_per_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=163840 expected_recv=261395 expected_overflow=0 exact_recv_per_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=163840 expected_recv=263245 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=163840 expected_recv=262567 expected_overflow=0 exact_recv_per_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=163840 expected_recv=261395 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=163840 expected_recv=261951 expected_overflow=0 exact_recv_per_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=163840 expected_recv=262567 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=163840 expected_recv=262216 expected_overflow=0 exact_recv_per_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=163840 expected_recv=261951 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=163840 expected_recv=262505 expected_overflow=0 exact_recv_per_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=163840 expected_recv=262216 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=163840 expected_recv=262505 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=163840 expected_recv=262234 expected_overflow=0 exact_recv_per_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=163840 expected_recv=262234 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=163840 expected_recv=262156 expected_overflow=0 exact_recv_per_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=163840 expected_recv=262399 expected_overflow=0 exact_recv_per_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=163840 expected_recv=262156 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=163840 expected_recv=262399 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=163840 expected_recv=261598 expected_overflow=0 exact_recv_per_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=163840 expected_recv=262514 expected_overflow=0 exact_recv_per_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=163840 expected_recv=261598 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=163840 expected_recv=262514 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=163840 expected_recv=261905 expected_overflow=0 exact_recv_per_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=163840 expected_recv=261905 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=163840 expected_recv=262246 expected_overflow=0 exact_recv_per_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=163840 expected_recv=262246 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=163840 expected_recv=261866 expected_overflow=0 exact_recv_per_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=163840 expected_recv=261866 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=163840 expected_recv=261906 expected_overflow=0 exact_recv_per_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=163840 expected_recv=261906 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[130580, 131286, 130924, 131289, 130888, 131501, 131342, 130766] avg=131072.0000 max=131501 max_rank=5 min=130580 min_rank=0 imbalance_ratio=1.003273 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[131074, 130805, 131400, 131359, 131315, 130739, 130820, 131064] avg=131072.0000 max=131400 max_rank=2 min=130739 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[130605, 130907, 131116, 131384, 130667, 131472, 131053, 131372] avg=131072.0000 max=131472 max_rank=5 min=130605 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[130893, 131221, 130571, 131041, 131238, 131277, 131177, 131158] avg=131072.0000 max=131277 max_rank=5 min=130571 min_rank=2 imbalance_ratio=1.001564 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[130766, 130827, 131148, 131639, 131052, 130823, 131173, 131148] avg=131072.0000 max=131639 max_rank=3 min=130766 min_rank=0 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[131011, 131514, 131305, 131039, 131193, 130615, 130760, 131139] avg=131072.0000 max=131514 max_rank=1 min=130615 min_rank=5 imbalance_ratio=1.003372 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[130888, 131437, 131229, 130780, 131413, 131464, 130458, 130907] avg=131072.0000 max=131464 max_rank=5 min=130458 min_rank=6 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[130631, 131098, 131091, 131221, 131473, 130745, 131208, 131109] avg=131072.0000 max=131473 max_rank=4 min=130631 min_rank=0 imbalance_ratio=1.003059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[130653, 130957, 131176, 131089, 130789, 131497, 131319, 131096] avg=131072.0000 max=131497 max_rank=5 min=130653 min_rank=0 imbalance_ratio=1.003242 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[130972, 131328, 131355, 130703, 131296, 130913, 130775, 131234] avg=131072.0000 max=131355 max_rank=2 min=130703 min_rank=3 imbalance_ratio=1.002159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[131050, 130881, 130941, 130638, 131504, 131238, 131159, 131165] avg=131072.0000 max=131504 max_rank=4 min=130638 min_rank=3 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[131591, 131019, 131436, 131202, 130761, 130491, 131274, 130802] avg=131072.0000 max=131591 max_rank=0 min=130491 min_rank=5 imbalance_ratio=1.003960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[130825, 130780, 131336, 131677, 131382, 130438, 131375, 130763] avg=131072.0000 max=131677 max_rank=3 min=130438 min_rank=5 imbalance_ratio=1.004616 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[130961, 131182, 131036, 131214, 130915, 130905, 131239, 131124] avg=131072.0000 max=131239 max_rank=6 min=130905 min_rank=5 imbalance_ratio=1.001274 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[130681, 131079, 131905, 131038, 130957, 131221, 131346, 130349] avg=131072.0000 max=131905 max_rank=2 min=130349 min_rank=7 imbalance_ratio=1.006355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=4190539 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1019904 peak_demand=131591 required_capacity=1.003960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[131383, 131045, 131055, 130487, 131053, 131154, 131259, 131140] avg=131072.0000 max=131383 max_rank=0 min=130487 min_rank=3 imbalance_ratio=1.002373 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=4194154 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016832 peak_demand=131398 required_capacity=1.002487 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=4196744 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1012608 peak_demand=131560 required_capacity=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=4197121 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014016 peak_demand=131905 required_capacity=1.006355 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=4194572 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015552 peak_demand=131821 required_capacity=1.005714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=4193384 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016192 peak_demand=131677 required_capacity=1.004616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=4190706 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1020032 peak_demand=131501 required_capacity=1.003273 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=4197212 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1012736 peak_demand=131734 required_capacity=1.005051 + [2026-08-30 05:20:21] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 26881.7 | throughput per GPU (TFLOP/s/GPU): 12.4 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102526E+01 | loss scale: 1.0 | grad norm: 44.093 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 132352 | min safe capacity: 1.0098 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58074.8916015625 | max allocated: 86257.341796875 | reserved: 87986.0 | max reserved: 87986.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262791 expected_overflow=0 exact_recv_per_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=163840 expected_recv=261917 expected_overflow=0 exact_recv_per_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=163840 expected_recv=261609 expected_overflow=0 exact_recv_per_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=163840 expected_recv=261873 expected_overflow=0 exact_recv_per_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262769 expected_overflow=0 exact_recv_per_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262791 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262038 expected_overflow=0 exact_recv_per_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262262 expected_overflow=0 exact_recv_per_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=163840 expected_recv=261893 expected_overflow=0 exact_recv_per_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=163840 expected_recv=261917 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=163840 expected_recv=261609 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=163840 expected_recv=261873 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262769 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262038 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=163840 expected_recv=261893 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=163840 expected_recv=262262 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262047 expected_overflow=0 exact_recv_per_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262047 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262515 expected_overflow=0 exact_recv_per_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262288 expected_overflow=0 exact_recv_per_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262561 expected_overflow=0 exact_recv_per_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=163840 expected_recv=261210 expected_overflow=0 exact_recv_per_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=163840 expected_recv=261450 expected_overflow=0 exact_recv_per_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262515 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262288 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262561 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=163840 expected_recv=261210 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=163840 expected_recv=261450 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262892 expected_overflow=0 exact_recv_per_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262189 expected_overflow=0 exact_recv_per_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262892 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=163840 expected_recv=262082 expected_overflow=0 exact_recv_per_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=163840 expected_recv=262189 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=163840 expected_recv=262082 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=163840 expected_recv=262313 expected_overflow=0 exact_recv_per_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=163840 expected_recv=261970 expected_overflow=0 exact_recv_per_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=163840 expected_recv=262250 expected_overflow=0 exact_recv_per_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=163840 expected_recv=262827 expected_overflow=0 exact_recv_per_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=163840 expected_recv=262313 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=163840 expected_recv=261985 expected_overflow=0 exact_recv_per_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=163840 expected_recv=261970 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=163840 expected_recv=262827 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=163840 expected_recv=262250 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=163840 expected_recv=261985 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=163840 expected_recv=261994 expected_overflow=0 exact_recv_per_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=163840 expected_recv=261731 expected_overflow=0 exact_recv_per_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=163840 expected_recv=261718 expected_overflow=0 exact_recv_per_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=163840 expected_recv=261994 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=163840 expected_recv=261731 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=163840 expected_recv=261718 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262642 expected_overflow=0 exact_recv_per_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=163840 expected_recv=261360 expected_overflow=0 exact_recv_per_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262103 expected_overflow=0 exact_recv_per_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262458 expected_overflow=0 exact_recv_per_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262642 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262419 expected_overflow=0 exact_recv_per_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=163840 expected_recv=261360 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262103 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262298 expected_overflow=0 exact_recv_per_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262458 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262419 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=163840 expected_recv=262273 expected_overflow=0 exact_recv_per_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262154 expected_overflow=0 exact_recv_per_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262298 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=163840 expected_recv=262273 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=163840 expected_recv=262154 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=163840 expected_recv=261677 expected_overflow=0 exact_recv_per_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=163840 expected_recv=262327 expected_overflow=0 exact_recv_per_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=163840 expected_recv=262848 expected_overflow=0 exact_recv_per_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=163840 expected_recv=261677 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=163840 expected_recv=261508 expected_overflow=0 exact_recv_per_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=163840 expected_recv=261658 expected_overflow=0 exact_recv_per_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=163840 expected_recv=263277 expected_overflow=0 exact_recv_per_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=163840 expected_recv=262327 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=163840 expected_recv=262848 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=163840 expected_recv=261508 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=163840 expected_recv=261658 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=163840 expected_recv=263277 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=163840 expected_recv=261592 expected_overflow=0 exact_recv_per_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=163840 expected_recv=261584 expected_overflow=0 exact_recv_per_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=163840 expected_recv=261592 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=163840 expected_recv=261584 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262249 expected_overflow=0 exact_recv_per_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262005 expected_overflow=0 exact_recv_per_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262233 expected_overflow=0 exact_recv_per_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262249 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262117 expected_overflow=0 exact_recv_per_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262005 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262233 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262198 expected_overflow=0 exact_recv_per_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262260 expected_overflow=0 exact_recv_per_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262117 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262585 expected_overflow=0 exact_recv_per_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262198 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262260 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262498 expected_overflow=0 exact_recv_per_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262585 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=163840 expected_recv=262498 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262093 expected_overflow=0 exact_recv_per_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=163840 expected_recv=261555 expected_overflow=0 exact_recv_per_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262093 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262228 expected_overflow=0 exact_recv_per_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=163840 expected_recv=261555 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262228 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262063 expected_overflow=0 exact_recv_per_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262366 expected_overflow=0 exact_recv_per_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262732 expected_overflow=0 exact_recv_per_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=163840 expected_recv=261400 expected_overflow=0 exact_recv_per_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=163840 expected_recv=261530 expected_overflow=0 exact_recv_per_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262366 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262063 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=163840 expected_recv=262732 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=163840 expected_recv=261400 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=163840 expected_recv=261530 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262347 expected_overflow=0 exact_recv_per_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262347 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=163840 expected_recv=261669 expected_overflow=0 exact_recv_per_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=163840 expected_recv=261971 expected_overflow=0 exact_recv_per_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=163840 expected_recv=261669 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=163840 expected_recv=261971 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262318 expected_overflow=0 exact_recv_per_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262334 expected_overflow=0 exact_recv_per_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262455 expected_overflow=0 exact_recv_per_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=163840 expected_recv=262095 expected_overflow=0 exact_recv_per_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262658 expected_overflow=0 exact_recv_per_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262318 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262334 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262455 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=163840 expected_recv=262658 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=163840 expected_recv=262095 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=163840 expected_recv=262634 expected_overflow=0 exact_recv_per_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=163840 expected_recv=262576 expected_overflow=0 exact_recv_per_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=163840 expected_recv=262634 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=163840 expected_recv=261782 expected_overflow=0 exact_recv_per_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=163840 expected_recv=262576 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=163840 expected_recv=261782 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=163840 expected_recv=262269 expected_overflow=0 exact_recv_per_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=163840 expected_recv=261580 expected_overflow=0 exact_recv_per_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=163840 expected_recv=262269 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=163840 expected_recv=261728 expected_overflow=0 exact_recv_per_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=163840 expected_recv=261580 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=163840 expected_recv=262488 expected_overflow=0 exact_recv_per_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262266 expected_overflow=0 exact_recv_per_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=163840 expected_recv=261728 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=163840 expected_recv=262488 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262266 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=163840 expected_recv=261873 expected_overflow=0 exact_recv_per_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262487 expected_overflow=0 exact_recv_per_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=163840 expected_recv=261873 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262456 expected_overflow=0 exact_recv_per_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262487 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262456 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262038 expected_overflow=0 exact_recv_per_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262166 expected_overflow=0 exact_recv_per_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262038 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=163840 expected_recv=261383 expected_overflow=0 exact_recv_per_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262483 expected_overflow=0 exact_recv_per_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262166 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=163840 expected_recv=261402 expected_overflow=0 exact_recv_per_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=163840 expected_recv=261383 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=163840 expected_recv=262483 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=163840 expected_recv=261402 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=163840 expected_recv=262609 expected_overflow=0 exact_recv_per_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=163840 expected_recv=262608 expected_overflow=0 exact_recv_per_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=163840 expected_recv=262609 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=163840 expected_recv=262788 expected_overflow=0 exact_recv_per_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=163840 expected_recv=262608 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=163840 expected_recv=262788 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=163840 expected_recv=262182 expected_overflow=0 exact_recv_per_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=163840 expected_recv=261864 expected_overflow=0 exact_recv_per_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=163840 expected_recv=262182 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=163840 expected_recv=261961 expected_overflow=0 exact_recv_per_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=163840 expected_recv=261738 expected_overflow=0 exact_recv_per_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=163840 expected_recv=262657 expected_overflow=0 exact_recv_per_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=163840 expected_recv=261864 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=163840 expected_recv=261961 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=163840 expected_recv=261738 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=163840 expected_recv=262657 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=163840 expected_recv=262329 expected_overflow=0 exact_recv_per_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=163840 expected_recv=262646 expected_overflow=0 exact_recv_per_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=163840 expected_recv=262036 expected_overflow=0 exact_recv_per_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=163840 expected_recv=262329 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=163840 expected_recv=262646 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=163840 expected_recv=262036 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=163840 expected_recv=261781 expected_overflow=0 exact_recv_per_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=163840 expected_recv=262335 expected_overflow=0 exact_recv_per_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=163840 expected_recv=261781 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=163840 expected_recv=261717 expected_overflow=0 exact_recv_per_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=163840 expected_recv=262335 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=163840 expected_recv=261314 expected_overflow=0 exact_recv_per_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=163840 expected_recv=261717 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=163840 expected_recv=261651 expected_overflow=0 exact_recv_per_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=163840 expected_recv=261314 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=163840 expected_recv=261651 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=163840 expected_recv=261913 expected_overflow=0 exact_recv_per_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=163840 expected_recv=262050 expected_overflow=0 exact_recv_per_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=163840 expected_recv=262462 expected_overflow=0 exact_recv_per_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=163840 expected_recv=261913 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=163840 expected_recv=262050 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=163840 expected_recv=261772 expected_overflow=0 exact_recv_per_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=163840 expected_recv=262462 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=163840 expected_recv=261772 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=163840 expected_recv=262220 expected_overflow=0 exact_recv_per_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=163840 expected_recv=262047 expected_overflow=0 exact_recv_per_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=163840 expected_recv=262220 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262085 expected_overflow=0 exact_recv_per_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=163840 expected_recv=262047 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=163840 expected_recv=263374 expected_overflow=0 exact_recv_per_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262085 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=163840 expected_recv=263374 actual_valid_recv=265472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262351 expected_overflow=0 exact_recv_per_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262045 expected_overflow=0 exact_recv_per_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=265472 slack_slots=62208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262597 expected_overflow=0 exact_recv_per_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262351 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262151 expected_overflow=0 exact_recv_per_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262045 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262597 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262151 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262221 expected_overflow=0 exact_recv_per_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=163840 expected_recv=261820 expected_overflow=0 exact_recv_per_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=163840 expected_recv=261995 expected_overflow=0 exact_recv_per_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=163840 expected_recv=262221 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=163840 expected_recv=261820 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=163840 expected_recv=261995 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=163840 expected_recv=261882 expected_overflow=0 exact_recv_per_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262147 expected_overflow=0 exact_recv_per_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=163840 expected_recv=261882 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262018 expected_overflow=0 exact_recv_per_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=163840 expected_recv=261871 expected_overflow=0 exact_recv_per_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262147 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262018 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262756 expected_overflow=0 exact_recv_per_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=163840 expected_recv=261871 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=163840 expected_recv=261587 expected_overflow=0 exact_recv_per_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262756 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262597 expected_overflow=0 exact_recv_per_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262138 expected_overflow=0 exact_recv_per_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=163840 expected_recv=261587 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262597 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262138 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262181 expected_overflow=0 exact_recv_per_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=163840 expected_recv=261956 expected_overflow=0 exact_recv_per_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=163840 expected_recv=261910 expected_overflow=0 exact_recv_per_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=163840 expected_recv=262181 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262213 expected_overflow=0 exact_recv_per_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262142 expected_overflow=0 exact_recv_per_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=163840 expected_recv=261956 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=163840 expected_recv=261910 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262213 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262142 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262044 expected_overflow=0 exact_recv_per_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262299 expected_overflow=0 exact_recv_per_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262044 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262299 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262450 expected_overflow=0 exact_recv_per_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=163840 expected_recv=262450 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[130656, 131637, 131294, 130982, 130612, 130585, 131258, 131552] avg=131072.0000 max=131637 max_rank=1 min=130585 min_rank=5 imbalance_ratio=1.004311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[131296, 130759, 131127, 131018, 130721, 131188, 131362, 131105] avg=131072.0000 max=131362 max_rank=6 min=130721 min_rank=4 imbalance_ratio=1.002213 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[131159, 131290, 130990, 130784, 131010, 131250, 131188, 130905] avg=131072.0000 max=131290 max_rank=1 min=130784 min_rank=3 imbalance_ratio=1.001663 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[131292, 130625, 131185, 130936, 131034, 131378, 131255, 130871] avg=131072.0000 max=131378 max_rank=5 min=130625 min_rank=1 imbalance_ratio=1.002335 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[130931, 131417, 130835, 130855, 130745, 131273, 131660, 130860] avg=131072.0000 max=131660 max_rank=6 min=130745 min_rank=4 imbalance_ratio=1.004486 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[131298, 131285, 131472, 130829, 131128, 130974, 130826, 130764] avg=131072.0000 max=131472 max_rank=2 min=130764 min_rank=7 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[130906, 130682, 130659, 131506, 131264, 131145, 131387, 131027] avg=131072.0000 max=131506 max_rank=3 min=130659 min_rank=2 imbalance_ratio=1.003311 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[131242, 131066, 131228, 131054, 130720, 131066, 131524, 130676] avg=131072.0000 max=131524 max_rank=6 min=130676 min_rank=7 imbalance_ratio=1.003448 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[131362, 131173, 131082, 130873, 130921, 131400, 130687, 131078] avg=131072.0000 max=131400 max_rank=5 min=130687 min_rank=6 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[131440, 131311, 131381, 130256, 131532, 130623, 131123, 130910] avg=131072.0000 max=131532 max_rank=4 min=130256 min_rank=3 imbalance_ratio=1.003510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[131114, 131702, 131169, 130769, 131489, 130899, 130747, 130687] avg=131072.0000 max=131702 max_rank=1 min=130687 min_rank=7 imbalance_ratio=1.004807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[130717, 130881, 130894, 131004, 130827, 131110, 131761, 131382] avg=131072.0000 max=131761 max_rank=6 min=130717 min_rank=0 imbalance_ratio=1.005257 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[131111, 130898, 130686, 131922, 131246, 130754, 131035, 130924] avg=131072.0000 max=131922 max_rank=3 min=130686 min_rank=2 imbalance_ratio=1.006485 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[130933, 131073, 130807, 130963, 131137, 131565, 131348, 130750] avg=131072.0000 max=131565 max_rank=5 min=130750 min_rank=7 imbalance_ratio=1.003761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[131306, 131204, 131128, 131060, 130697, 130996, 131110, 131075] avg=131072.0000 max=131306 max_rank=0 min=130697 min_rank=4 imbalance_ratio=1.001785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[131183, 130990, 130609, 131382, 131122, 131102, 131193, 130995] avg=131072.0000 max=131382 max_rank=3 min=130609 min_rank=2 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=4192440 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016960 peak_demand=131558 required_capacity=1.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=4194187 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016832 peak_demand=131532 required_capacity=1.003510 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=4194583 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016960 peak_demand=131765 required_capacity=1.005287 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=4194653 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014528 peak_demand=131450 required_capacity=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=4193763 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016960 peak_demand=131472 required_capacity=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=4195970 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1012864 peak_demand=131710 required_capacity=1.004868 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=4195524 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015040 peak_demand=131761 required_capacity=1.005257 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=4193312 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017984 peak_demand=131922 required_capacity=1.006485 + [2026-08-30 05:20:21] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 728.5 | throughput per GPU (TFLOP/s/GPU): 457.6 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101901E+01 | loss scale: 1.0 | grad norm: 45.911 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 132736 | min safe capacity: 1.0127 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=163840 expected_recv=261176 expected_overflow=0 exact_recv_per_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262038 expected_overflow=0 exact_recv_per_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262428 expected_overflow=0 exact_recv_per_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=163840 expected_recv=261936 expected_overflow=0 exact_recv_per_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262177 expected_overflow=0 exact_recv_per_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262861 expected_overflow=0 exact_recv_per_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262589 expected_overflow=0 exact_recv_per_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=163840 expected_recv=261176 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262038 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=163840 expected_recv=261947 expected_overflow=0 exact_recv_per_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262428 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=163840 expected_recv=261936 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262177 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262861 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=163840 expected_recv=262589 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=163840 expected_recv=261947 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=163840 expected_recv=261945 expected_overflow=0 exact_recv_per_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=163840 expected_recv=262292 expected_overflow=0 exact_recv_per_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=163840 expected_recv=261867 expected_overflow=0 exact_recv_per_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=163840 expected_recv=261945 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=163840 expected_recv=262138 expected_overflow=0 exact_recv_per_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=163840 expected_recv=262832 expected_overflow=0 exact_recv_per_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=163840 expected_recv=262292 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=163840 expected_recv=262461 expected_overflow=0 exact_recv_per_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=163840 expected_recv=262281 expected_overflow=0 exact_recv_per_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=163840 expected_recv=261336 expected_overflow=0 exact_recv_per_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=163840 expected_recv=261867 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=163840 expected_recv=262832 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=163840 expected_recv=262138 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=163840 expected_recv=262461 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=163840 expected_recv=262281 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=163840 expected_recv=261336 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=163840 expected_recv=261879 expected_overflow=0 exact_recv_per_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262494 expected_overflow=0 exact_recv_per_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262528 expected_overflow=0 exact_recv_per_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=163840 expected_recv=261879 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262149 expected_overflow=0 exact_recv_per_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262216 expected_overflow=0 exact_recv_per_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262518 expected_overflow=0 exact_recv_per_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262186 expected_overflow=0 exact_recv_per_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262494 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=163840 expected_recv=261182 expected_overflow=0 exact_recv_per_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262149 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262528 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262216 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262518 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=163840 expected_recv=262186 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=163840 expected_recv=261182 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=163840 expected_recv=262372 expected_overflow=0 exact_recv_per_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=163840 expected_recv=261207 expected_overflow=0 exact_recv_per_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=163840 expected_recv=262471 expected_overflow=0 exact_recv_per_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=163840 expected_recv=262372 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=163840 expected_recv=261761 expected_overflow=0 exact_recv_per_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=163840 expected_recv=261207 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=163840 expected_recv=262120 expected_overflow=0 exact_recv_per_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=163840 expected_recv=262693 expected_overflow=0 exact_recv_per_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=163840 expected_recv=261744 expected_overflow=0 exact_recv_per_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=163840 expected_recv=262471 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=163840 expected_recv=261761 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=163840 expected_recv=262784 expected_overflow=0 exact_recv_per_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=163840 expected_recv=262120 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=163840 expected_recv=262693 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=163840 expected_recv=261744 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=163840 expected_recv=262784 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=163840 expected_recv=261506 expected_overflow=0 exact_recv_per_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=163840 expected_recv=262655 expected_overflow=0 exact_recv_per_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=163840 expected_recv=261506 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=163840 expected_recv=262655 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=163840 expected_recv=262322 expected_overflow=0 exact_recv_per_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=163840 expected_recv=262470 expected_overflow=0 exact_recv_per_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=163840 expected_recv=261652 expected_overflow=0 exact_recv_per_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=163840 expected_recv=261583 expected_overflow=0 exact_recv_per_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=163840 expected_recv=262322 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=163840 expected_recv=262470 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=163840 expected_recv=262698 expected_overflow=0 exact_recv_per_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=163840 expected_recv=262266 expected_overflow=0 exact_recv_per_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=163840 expected_recv=261652 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=163840 expected_recv=261583 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=163840 expected_recv=262698 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=163840 expected_recv=262266 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=163840 expected_recv=261794 expected_overflow=0 exact_recv_per_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=163840 expected_recv=261761 expected_overflow=0 exact_recv_per_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=163840 expected_recv=261794 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=163840 expected_recv=261761 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=163840 expected_recv=262535 expected_overflow=0 exact_recv_per_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=163840 expected_recv=262218 expected_overflow=0 exact_recv_per_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=163840 expected_recv=261963 expected_overflow=0 exact_recv_per_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=163840 expected_recv=261509 expected_overflow=0 exact_recv_per_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=163840 expected_recv=262535 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=163840 expected_recv=262218 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=163840 expected_recv=262808 expected_overflow=0 exact_recv_per_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=163840 expected_recv=261963 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=163840 expected_recv=262564 expected_overflow=0 exact_recv_per_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=163840 expected_recv=261509 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=163840 expected_recv=262808 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=163840 expected_recv=262564 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262308 expected_overflow=0 exact_recv_per_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262531 expected_overflow=0 exact_recv_per_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262308 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262531 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262135 expected_overflow=0 exact_recv_per_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262073 expected_overflow=0 exact_recv_per_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262266 expected_overflow=0 exact_recv_per_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=163840 expected_recv=261868 expected_overflow=0 exact_recv_per_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262135 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262266 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262073 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=163840 expected_recv=261868 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=163840 expected_recv=261771 expected_overflow=0 exact_recv_per_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262200 expected_overflow=0 exact_recv_per_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=163840 expected_recv=262470 expected_overflow=0 exact_recv_per_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=163840 expected_recv=261771 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=163840 expected_recv=261906 expected_overflow=0 exact_recv_per_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=163840 expected_recv=262200 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=163840 expected_recv=262470 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=163840 expected_recv=261906 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=163840 expected_recv=262053 expected_overflow=0 exact_recv_per_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=163840 expected_recv=262510 expected_overflow=0 exact_recv_per_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=163840 expected_recv=262053 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=163840 expected_recv=262567 expected_overflow=0 exact_recv_per_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=163840 expected_recv=262510 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=163840 expected_recv=261598 expected_overflow=0 exact_recv_per_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=163840 expected_recv=262567 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=163840 expected_recv=261893 expected_overflow=0 exact_recv_per_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=163840 expected_recv=262155 expected_overflow=0 exact_recv_per_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=163840 expected_recv=261598 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=163840 expected_recv=262655 expected_overflow=0 exact_recv_per_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=163840 expected_recv=262500 expected_overflow=0 exact_recv_per_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=163840 expected_recv=261893 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=163840 expected_recv=262155 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=163840 expected_recv=262655 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=163840 expected_recv=262500 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=163840 expected_recv=261924 expected_overflow=0 exact_recv_per_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=163840 expected_recv=261924 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=163840 expected_recv=261424 expected_overflow=0 exact_recv_per_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=163840 expected_recv=262111 expected_overflow=0 exact_recv_per_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=163840 expected_recv=262125 expected_overflow=0 exact_recv_per_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=163840 expected_recv=261424 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=163840 expected_recv=262111 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=163840 expected_recv=263134 expected_overflow=0 exact_recv_per_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262317 expected_overflow=0 exact_recv_per_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=163840 expected_recv=262125 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=163840 expected_recv=261279 expected_overflow=0 exact_recv_per_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262370 expected_overflow=0 exact_recv_per_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=163840 expected_recv=263134 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262317 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=163840 expected_recv=261279 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262370 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=163840 expected_recv=261964 expected_overflow=0 exact_recv_per_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=163840 expected_recv=261964 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262308 expected_overflow=0 exact_recv_per_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262422 expected_overflow=0 exact_recv_per_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=163840 expected_recv=261801 expected_overflow=0 exact_recv_per_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262308 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262422 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=163840 expected_recv=261801 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262187 expected_overflow=0 exact_recv_per_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=163840 expected_recv=261878 expected_overflow=0 exact_recv_per_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262187 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262064 expected_overflow=0 exact_recv_per_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=163840 expected_recv=261878 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262092 expected_overflow=0 exact_recv_per_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262064 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262286 expected_overflow=0 exact_recv_per_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=163840 expected_recv=262092 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=163840 expected_recv=261834 expected_overflow=0 exact_recv_per_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262286 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262510 expected_overflow=0 exact_recv_per_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=163840 expected_recv=261834 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262935 expected_overflow=0 exact_recv_per_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262510 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=163840 expected_recv=261642 expected_overflow=0 exact_recv_per_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=163840 expected_recv=262935 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=163840 expected_recv=261538 expected_overflow=0 exact_recv_per_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=163840 expected_recv=261642 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=163840 expected_recv=261857 expected_overflow=0 exact_recv_per_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=163840 expected_recv=261798 expected_overflow=0 exact_recv_per_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=163840 expected_recv=261538 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=163840 expected_recv=261857 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=163840 expected_recv=261798 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=163840 expected_recv=262125 expected_overflow=0 exact_recv_per_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=163840 expected_recv=262147 expected_overflow=0 exact_recv_per_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=163840 expected_recv=262125 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=163840 expected_recv=262945 expected_overflow=0 exact_recv_per_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=163840 expected_recv=262147 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=163840 expected_recv=262218 expected_overflow=0 exact_recv_per_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=163840 expected_recv=262945 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=163840 expected_recv=262218 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=163840 expected_recv=262269 expected_overflow=0 exact_recv_per_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=163840 expected_recv=261706 expected_overflow=0 exact_recv_per_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=163840 expected_recv=261949 expected_overflow=0 exact_recv_per_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262561 expected_overflow=0 exact_recv_per_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=163840 expected_recv=262269 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=163840 expected_recv=261706 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=163840 expected_recv=261949 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262561 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262221 expected_overflow=0 exact_recv_per_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262126 expected_overflow=0 exact_recv_per_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262221 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262378 expected_overflow=0 exact_recv_per_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262126 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=163840 expected_recv=261771 expected_overflow=0 exact_recv_per_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262378 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=163840 expected_recv=261771 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262108 expected_overflow=0 exact_recv_per_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=163840 expected_recv=261804 expected_overflow=0 exact_recv_per_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=163840 expected_recv=262581 expected_overflow=0 exact_recv_per_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262108 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262281 expected_overflow=0 exact_recv_per_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=163840 expected_recv=261804 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=163840 expected_recv=262581 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=163840 expected_recv=262281 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=163840 expected_recv=262159 expected_overflow=0 exact_recv_per_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=163840 expected_recv=262359 expected_overflow=0 exact_recv_per_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=163840 expected_recv=262159 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=163840 expected_recv=261604 expected_overflow=0 exact_recv_per_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=163840 expected_recv=262359 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=163840 expected_recv=261854 expected_overflow=0 exact_recv_per_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=163840 expected_recv=261604 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=163840 expected_recv=261854 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=163840 expected_recv=262716 expected_overflow=0 exact_recv_per_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262375 expected_overflow=0 exact_recv_per_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262246 expected_overflow=0 exact_recv_per_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=163840 expected_recv=262075 expected_overflow=0 exact_recv_per_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=163840 expected_recv=262716 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262375 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262246 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262563 expected_overflow=0 exact_recv_per_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=163840 expected_recv=262075 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262001 expected_overflow=0 exact_recv_per_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262563 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=163840 expected_recv=261884 expected_overflow=0 exact_recv_per_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262001 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=163840 expected_recv=261884 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=163840 expected_recv=261643 expected_overflow=0 exact_recv_per_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262386 expected_overflow=0 exact_recv_per_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262386 expected_overflow=0 exact_recv_per_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=163840 expected_recv=261643 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=163840 expected_recv=261609 expected_overflow=0 exact_recv_per_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262054 expected_overflow=0 exact_recv_per_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262386 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262386 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=163840 expected_recv=261609 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=163840 expected_recv=261931 expected_overflow=0 exact_recv_per_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=163840 expected_recv=262054 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=163840 expected_recv=261931 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262069 expected_overflow=0 exact_recv_per_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262069 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262193 expected_overflow=0 exact_recv_per_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=163840 expected_recv=261469 expected_overflow=0 exact_recv_per_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262765 expected_overflow=0 exact_recv_per_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262193 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=163840 expected_recv=261469 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262765 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262730 expected_overflow=0 exact_recv_per_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=163840 expected_recv=262730 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[131134, 130834, 130374, 131078, 131229, 131766, 130926, 131235] avg=131072.0000 max=131766 max_rank=5 min=130374 min_rank=2 imbalance_ratio=1.005295 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[130917, 130564, 131057, 131529, 130730, 131545, 131412, 130822] avg=131072.0000 max=131545 max_rank=5 min=130564 min_rank=1 imbalance_ratio=1.003609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[130694, 131465, 130881, 130847, 130817, 131156, 131225, 131491] avg=131072.0000 max=131491 max_rank=7 min=130694 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[130923, 131121, 131265, 130785, 131364, 131436, 130830, 130852] avg=131072.0000 max=131436 max_rank=5 min=130785 min_rank=3 imbalance_ratio=1.002777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[131214, 131366, 130874, 131252, 131008, 130779, 130979, 131104] avg=131072.0000 max=131366 max_rank=1 min=130779 min_rank=5 imbalance_ratio=1.002243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[131203, 131467, 130908, 130852, 131365, 130851, 131056, 130874] avg=131072.0000 max=131467 max_rank=1 min=130851 min_rank=5 imbalance_ratio=1.003014 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[131385, 131025, 131167, 131311, 131043, 130926, 130894, 130825] avg=131072.0000 max=131385 max_rank=0 min=130825 min_rank=7 imbalance_ratio=1.002388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[130958, 130634, 131105, 130925, 131461, 130928, 130947, 131618] avg=131072.0000 max=131618 max_rank=7 min=130634 min_rank=1 imbalance_ratio=1.004166 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[131290, 131735, 131031, 130963, 130619, 130958, 130771, 131209] avg=131072.0000 max=131735 max_rank=1 min=130619 min_rank=4 imbalance_ratio=1.005058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[131292, 130808, 130824, 131305, 130925, 130757, 131452, 131213] avg=131072.0000 max=131452 max_rank=6 min=130757 min_rank=5 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[131256, 130815, 131155, 131386, 130553, 131413, 130672, 131326] avg=131072.0000 max=131413 max_rank=5 min=130553 min_rank=4 imbalance_ratio=1.002602 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[130944, 131196, 130932, 131527, 131024, 131184, 130860, 130909] avg=131072.0000 max=131527 max_rank=3 min=130860 min_rank=6 imbalance_ratio=1.003471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[131167, 131384, 130770, 131521, 131315, 131097, 130702, 130620] avg=131072.0000 max=131521 max_rank=3 min=130620 min_rank=7 imbalance_ratio=1.003426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[130983, 130993, 131123, 130805, 131510, 131074, 131461, 130627] avg=131072.0000 max=131510 max_rank=4 min=130627 min_rank=7 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[131246, 131067, 131393, 130734, 131397, 130796, 130771, 131172] avg=131072.0000 max=131397 max_rank=4 min=130734 min_rank=3 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[130654, 131491, 131429, 130849, 131619, 130989, 130653, 130892] avg=131072.0000 max=131619 max_rank=4 min=130653 min_rank=6 imbalance_ratio=1.004173 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=4194296 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015424 peak_demand=131618 required_capacity=1.004166 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=4194651 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016320 peak_demand=131627 required_capacity=1.004234 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=4193759 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015680 peak_demand=131598 required_capacity=1.004013 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=4193373 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015936 peak_demand=131619 required_capacity=1.004173 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=4193567 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018496 peak_demand=131663 required_capacity=1.004509 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=4195317 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015040 peak_demand=131735 required_capacity=1.005058 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=4196170 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014656 peak_demand=131642 required_capacity=1.004349 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=4193299 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017728 peak_demand=131766 required_capacity=1.005295 + [2026-08-30 05:20:22] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 547.2 | throughput per GPU (TFLOP/s/GPU): 609.2 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.058692E+01 | loss scale: 1.0 | grad norm: 216.484 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 132736 | min safe capacity: 1.0127 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=163840 expected_recv=261503 expected_overflow=0 exact_recv_per_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262072 expected_overflow=0 exact_recv_per_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=163840 expected_recv=261956 expected_overflow=0 exact_recv_per_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262199 expected_overflow=0 exact_recv_per_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262299 expected_overflow=0 exact_recv_per_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262459 expected_overflow=0 exact_recv_per_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262072 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262422 expected_overflow=0 exact_recv_per_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=163840 expected_recv=261503 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=163840 expected_recv=261956 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262199 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262242 expected_overflow=0 exact_recv_per_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262299 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262459 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262422 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=163840 expected_recv=262242 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=163840 expected_recv=261865 expected_overflow=0 exact_recv_per_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262727 expected_overflow=0 exact_recv_per_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=163840 expected_recv=261865 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262125 expected_overflow=0 exact_recv_per_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262186 expected_overflow=0 exact_recv_per_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262727 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262198 expected_overflow=0 exact_recv_per_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262125 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262186 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262198 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262143 expected_overflow=0 exact_recv_per_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=163840 expected_recv=261720 expected_overflow=0 exact_recv_per_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262188 expected_overflow=0 exact_recv_per_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262143 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=163840 expected_recv=261934 expected_overflow=0 exact_recv_per_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=163840 expected_recv=261720 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=163840 expected_recv=262188 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=163840 expected_recv=261934 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=163840 expected_recv=262625 expected_overflow=0 exact_recv_per_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=163840 expected_recv=262552 expected_overflow=0 exact_recv_per_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=163840 expected_recv=262625 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=163840 expected_recv=261605 expected_overflow=0 exact_recv_per_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=163840 expected_recv=262552 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=163840 expected_recv=262019 expected_overflow=0 exact_recv_per_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=163840 expected_recv=261605 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=163840 expected_recv=262019 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=163840 expected_recv=262293 expected_overflow=0 exact_recv_per_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=163840 expected_recv=261411 expected_overflow=0 exact_recv_per_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=163840 expected_recv=262713 expected_overflow=0 exact_recv_per_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=163840 expected_recv=262293 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=163840 expected_recv=261411 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=163840 expected_recv=262713 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=163840 expected_recv=262634 expected_overflow=0 exact_recv_per_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=163840 expected_recv=262634 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=163840 expected_recv=261900 expected_overflow=0 exact_recv_per_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=163840 expected_recv=262447 expected_overflow=0 exact_recv_per_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=163840 expected_recv=261900 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=163840 expected_recv=262447 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=163840 expected_recv=261694 expected_overflow=0 exact_recv_per_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=163840 expected_recv=261900 expected_overflow=0 exact_recv_per_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=163840 expected_recv=261694 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=163840 expected_recv=261900 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=163840 expected_recv=262361 expected_overflow=0 exact_recv_per_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=163840 expected_recv=261760 expected_overflow=0 exact_recv_per_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=163840 expected_recv=262456 expected_overflow=0 exact_recv_per_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=163840 expected_recv=262361 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=163840 expected_recv=261760 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=163840 expected_recv=262456 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=163840 expected_recv=262533 expected_overflow=0 exact_recv_per_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=163840 expected_recv=262241 expected_overflow=0 exact_recv_per_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=163840 expected_recv=262533 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=163840 expected_recv=262722 expected_overflow=0 exact_recv_per_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=163840 expected_recv=262241 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=163840 expected_recv=262722 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=163840 expected_recv=261701 expected_overflow=0 exact_recv_per_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=163840 expected_recv=262225 expected_overflow=0 exact_recv_per_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=163840 expected_recv=261701 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=163840 expected_recv=261912 expected_overflow=0 exact_recv_per_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=163840 expected_recv=262225 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=163840 expected_recv=261742 expected_overflow=0 exact_recv_per_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=163840 expected_recv=262076 expected_overflow=0 exact_recv_per_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=163840 expected_recv=261912 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=163840 expected_recv=261742 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=163840 expected_recv=262076 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262598 expected_overflow=0 exact_recv_per_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262677 expected_overflow=0 exact_recv_per_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=163840 expected_recv=261326 expected_overflow=0 exact_recv_per_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262598 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262677 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=163840 expected_recv=261326 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=163840 expected_recv=261480 expected_overflow=0 exact_recv_per_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=163840 expected_recv=261480 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262169 expected_overflow=0 exact_recv_per_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262531 expected_overflow=0 exact_recv_per_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=163840 expected_recv=261794 expected_overflow=0 exact_recv_per_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262169 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262577 expected_overflow=0 exact_recv_per_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262531 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262108 expected_overflow=0 exact_recv_per_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=163840 expected_recv=261775 expected_overflow=0 exact_recv_per_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=163840 expected_recv=261794 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=163840 expected_recv=262577 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262525 expected_overflow=0 exact_recv_per_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262108 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=163840 expected_recv=261775 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=163840 expected_recv=261774 expected_overflow=0 exact_recv_per_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262525 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=163840 expected_recv=261774 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262183 expected_overflow=0 exact_recv_per_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262902 expected_overflow=0 exact_recv_per_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=163840 expected_recv=261790 expected_overflow=0 exact_recv_per_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262095 expected_overflow=0 exact_recv_per_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262902 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262183 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262351 expected_overflow=0 exact_recv_per_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262114 expected_overflow=0 exact_recv_per_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=163840 expected_recv=261790 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=163840 expected_recv=262095 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262351 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262114 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262152 expected_overflow=0 exact_recv_per_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262278 expected_overflow=0 exact_recv_per_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262152 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262278 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=163840 expected_recv=261953 expected_overflow=0 exact_recv_per_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=163840 expected_recv=261643 expected_overflow=0 exact_recv_per_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=163840 expected_recv=261953 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262852 expected_overflow=0 exact_recv_per_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=163840 expected_recv=261643 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=163840 expected_recv=261809 expected_overflow=0 exact_recv_per_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262768 expected_overflow=0 exact_recv_per_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262719 expected_overflow=0 exact_recv_per_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=163840 expected_recv=262852 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=163840 expected_recv=261809 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262768 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=163840 expected_recv=261695 expected_overflow=0 exact_recv_per_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262719 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=163840 expected_recv=261799 expected_overflow=0 exact_recv_per_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=163840 expected_recv=261695 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=163840 expected_recv=261799 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262013 expected_overflow=0 exact_recv_per_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262389 expected_overflow=0 exact_recv_per_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262013 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262219 expected_overflow=0 exact_recv_per_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262389 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=163840 expected_recv=261550 expected_overflow=0 exact_recv_per_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262140 expected_overflow=0 exact_recv_per_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262447 expected_overflow=0 exact_recv_per_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=163840 expected_recv=262219 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=163840 expected_recv=261550 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262110 expected_overflow=0 exact_recv_per_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262140 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262447 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262452 expected_overflow=0 exact_recv_per_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262110 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262452 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262653 expected_overflow=0 exact_recv_per_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262066 expected_overflow=0 exact_recv_per_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262653 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=163840 expected_recv=261731 expected_overflow=0 exact_recv_per_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=163840 expected_recv=261553 expected_overflow=0 exact_recv_per_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=163840 expected_recv=262066 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=163840 expected_recv=262214 expected_overflow=0 exact_recv_per_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=163840 expected_recv=261570 expected_overflow=0 exact_recv_per_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=163840 expected_recv=261731 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=163840 expected_recv=261553 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=163840 expected_recv=262214 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=163840 expected_recv=261761 expected_overflow=0 exact_recv_per_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=163840 expected_recv=261570 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=163840 expected_recv=262742 expected_overflow=0 exact_recv_per_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=163840 expected_recv=261761 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=163840 expected_recv=262742 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=163840 expected_recv=261993 expected_overflow=0 exact_recv_per_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=163840 expected_recv=261890 expected_overflow=0 exact_recv_per_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=163840 expected_recv=261993 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=163840 expected_recv=261890 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=163840 expected_recv=262895 expected_overflow=0 exact_recv_per_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=163840 expected_recv=262087 expected_overflow=0 exact_recv_per_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262612 expected_overflow=0 exact_recv_per_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262027 expected_overflow=0 exact_recv_per_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=163840 expected_recv=262895 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=163840 expected_recv=262087 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262612 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=163840 expected_recv=261874 expected_overflow=0 exact_recv_per_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262027 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262541 expected_overflow=0 exact_recv_per_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=163840 expected_recv=261874 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=163840 expected_recv=261544 expected_overflow=0 exact_recv_per_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262541 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262533 expected_overflow=0 exact_recv_per_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=163840 expected_recv=261544 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262533 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=163840 expected_recv=261927 expected_overflow=0 exact_recv_per_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262029 expected_overflow=0 exact_recv_per_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262094 expected_overflow=0 exact_recv_per_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=163840 expected_recv=261768 expected_overflow=0 exact_recv_per_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=163840 expected_recv=261927 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=163840 expected_recv=261980 expected_overflow=0 exact_recv_per_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262029 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=163840 expected_recv=262094 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=163840 expected_recv=261768 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262361 expected_overflow=0 exact_recv_per_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=163840 expected_recv=261980 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262360 expected_overflow=0 exact_recv_per_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262361 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262218 expected_overflow=0 exact_recv_per_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262360 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262218 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=163840 expected_recv=261701 expected_overflow=0 exact_recv_per_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=163840 expected_recv=262217 expected_overflow=0 exact_recv_per_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262735 expected_overflow=0 exact_recv_per_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=163840 expected_recv=262766 expected_overflow=0 exact_recv_per_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=163840 expected_recv=261701 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=163840 expected_recv=262217 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=163840 expected_recv=262331 expected_overflow=0 exact_recv_per_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=163840 expected_recv=262735 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=163840 expected_recv=262766 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=163840 expected_recv=261412 expected_overflow=0 exact_recv_per_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=163840 expected_recv=262331 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=163840 expected_recv=261830 expected_overflow=0 exact_recv_per_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=163840 expected_recv=261412 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=163840 expected_recv=262549 expected_overflow=0 exact_recv_per_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=163840 expected_recv=261830 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=163840 expected_recv=262549 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=163840 expected_recv=261910 expected_overflow=0 exact_recv_per_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=163840 expected_recv=262601 expected_overflow=0 exact_recv_per_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=163840 expected_recv=261446 expected_overflow=0 exact_recv_per_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=163840 expected_recv=261910 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=163840 expected_recv=262601 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=163840 expected_recv=261686 expected_overflow=0 exact_recv_per_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=163840 expected_recv=261446 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=163840 expected_recv=262050 expected_overflow=0 exact_recv_per_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=163840 expected_recv=261686 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=163840 expected_recv=262129 expected_overflow=0 exact_recv_per_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=163840 expected_recv=261959 expected_overflow=0 exact_recv_per_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=163840 expected_recv=262050 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=163840 expected_recv=262756 expected_overflow=0 exact_recv_per_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=163840 expected_recv=262129 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=163840 expected_recv=261959 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=163840 expected_recv=262806 expected_overflow=0 exact_recv_per_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=163840 expected_recv=262756 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=163840 expected_recv=262806 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=163840 expected_recv=262310 expected_overflow=0 exact_recv_per_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=163840 expected_recv=262352 expected_overflow=0 exact_recv_per_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=163840 expected_recv=262310 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=163840 expected_recv=262320 expected_overflow=0 exact_recv_per_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=163840 expected_recv=262165 expected_overflow=0 exact_recv_per_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=163840 expected_recv=262352 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=163840 expected_recv=262320 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=163840 expected_recv=262543 expected_overflow=0 exact_recv_per_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=163840 expected_recv=262165 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=163840 expected_recv=261893 expected_overflow=0 exact_recv_per_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=163840 expected_recv=261402 expected_overflow=0 exact_recv_per_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=163840 expected_recv=262543 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=163840 expected_recv=261893 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=163840 expected_recv=261402 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=163840 expected_recv=261966 expected_overflow=0 exact_recv_per_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=163840 expected_recv=262057 expected_overflow=0 exact_recv_per_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=163840 expected_recv=261966 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=163840 expected_recv=262057 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[131570, 131338, 130800, 130847, 131046, 131118, 131093, 130764] avg=131072.0000 max=131570 max_rank=0 min=130764 min_rank=7 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[130845, 130993, 131292, 130975, 131205, 131044, 131104, 131118] avg=131072.0000 max=131292 max_rank=2 min=130845 min_rank=0 imbalance_ratio=1.001678 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[130699, 131013, 131485, 130904, 131420, 131103, 130780, 131172] avg=131072.0000 max=131485 max_rank=2 min=130699 min_rank=0 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[131239, 131097, 131324, 130813, 131138, 130878, 131179, 130908] avg=131072.0000 max=131324 max_rank=2 min=130813 min_rank=3 imbalance_ratio=1.001923 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[131178, 131310, 130884, 131132, 131211, 130988, 130680, 131193] avg=131072.0000 max=131310 max_rank=1 min=130680 min_rank=6 imbalance_ratio=1.001816 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[130786, 131629, 130890, 130394, 131349, 130847, 131223, 131458] avg=131072.0000 max=131629 max_rank=1 min=130394 min_rank=3 imbalance_ratio=1.004250 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[130824, 131371, 131320, 130601, 130969, 130993, 131691, 130807] avg=131072.0000 max=131691 max_rank=6 min=130601 min_rank=3 imbalance_ratio=1.004723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[131536, 131337, 131269, 131219, 131067, 130154, 130535, 131459] avg=131072.0000 max=131536 max_rank=0 min=130154 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[131049, 131648, 130808, 130834, 130801, 131232, 130962, 131242] avg=131072.0000 max=131648 max_rank=1 min=130801 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[130705, 131358, 130937, 131127, 131037, 130771, 131449, 131192] avg=131072.0000 max=131449 max_rank=6 min=130705 min_rank=0 imbalance_ratio=1.002876 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[131558, 131081, 131133, 131111, 131265, 130792, 131097, 130539] avg=131072.0000 max=131558 max_rank=0 min=130539 min_rank=7 imbalance_ratio=1.003708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[130952, 130680, 131045, 131472, 130992, 130971, 131233, 131231] avg=131072.0000 max=131472 max_rank=3 min=130680 min_rank=1 imbalance_ratio=1.003052 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[130873, 130895, 131296, 130991, 131373, 131063, 130999, 131086] avg=131072.0000 max=131373 max_rank=4 min=130873 min_rank=0 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[131075, 131390, 131070, 130534, 130895, 131403, 131016, 131193] avg=131072.0000 max=131403 max_rank=5 min=130534 min_rank=3 imbalance_ratio=1.002525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[131086, 131041, 130922, 130797, 131188, 131605, 130856, 131081] avg=131072.0000 max=131605 max_rank=5 min=130797 min_rank=3 imbalance_ratio=1.004066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[130810, 131486, 131142, 130727, 131398, 130619, 131038, 131356] avg=131072.0000 max=131486 max_rank=1 min=130619 min_rank=5 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=4195650 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015296 peak_demand=131477 required_capacity=1.003090 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=4195784 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015168 peak_demand=131648 required_capacity=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=4193993 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015552 peak_demand=131838 required_capacity=1.005844 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=4195449 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014400 peak_demand=131691 required_capacity=1.004723 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=4194030 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015680 peak_demand=131420 required_capacity=1.002655 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=4192246 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018880 peak_demand=131631 required_capacity=1.004265 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=4194439 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016064 peak_demand=131605 required_capacity=1.004066 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=4192841 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1019264 peak_demand=131570 required_capacity=1.003799 + [2026-08-30 05:20:23] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 547.5 | throughput per GPU (TFLOP/s/GPU): 609.0 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.302340E+01 | loss scale: 1.0 | grad norm: 45.354 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 132736 | min safe capacity: 1.0127 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262221 expected_overflow=0 exact_recv_per_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262665 expected_overflow=0 exact_recv_per_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=163840 expected_recv=261447 expected_overflow=0 exact_recv_per_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262182 expected_overflow=0 exact_recv_per_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=163840 expected_recv=261869 expected_overflow=0 exact_recv_per_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262130 expected_overflow=0 exact_recv_per_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262125 expected_overflow=0 exact_recv_per_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262221 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262665 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=163840 expected_recv=261447 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262182 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262513 expected_overflow=0 exact_recv_per_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=163840 expected_recv=261869 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262130 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262125 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=163840 expected_recv=262513 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262019 expected_overflow=0 exact_recv_per_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262461 expected_overflow=0 exact_recv_per_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=163840 expected_recv=261709 expected_overflow=0 exact_recv_per_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262486 expected_overflow=0 exact_recv_per_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262019 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=163840 expected_recv=261977 expected_overflow=0 exact_recv_per_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262191 expected_overflow=0 exact_recv_per_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262461 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=163840 expected_recv=261709 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262486 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=163840 expected_recv=261798 expected_overflow=0 exact_recv_per_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262511 expected_overflow=0 exact_recv_per_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=163840 expected_recv=261977 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262191 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=163840 expected_recv=261798 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=163840 expected_recv=262511 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262375 expected_overflow=0 exact_recv_per_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262191 expected_overflow=0 exact_recv_per_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=163840 expected_recv=261962 expected_overflow=0 exact_recv_per_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=163840 expected_recv=261863 expected_overflow=0 exact_recv_per_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262375 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=163840 expected_recv=261877 expected_overflow=0 exact_recv_per_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262502 expected_overflow=0 exact_recv_per_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262191 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=163840 expected_recv=261962 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=163840 expected_recv=261863 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262198 expected_overflow=0 exact_recv_per_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=163840 expected_recv=261877 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262502 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262184 expected_overflow=0 exact_recv_per_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262198 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=163840 expected_recv=262184 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=163840 expected_recv=261339 expected_overflow=0 exact_recv_per_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262249 expected_overflow=0 exact_recv_per_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262524 expected_overflow=0 exact_recv_per_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=163840 expected_recv=261752 expected_overflow=0 exact_recv_per_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=163840 expected_recv=261339 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262570 expected_overflow=0 exact_recv_per_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262249 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=163840 expected_recv=261292 expected_overflow=0 exact_recv_per_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262524 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=163840 expected_recv=261752 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262855 expected_overflow=0 exact_recv_per_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262570 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=163840 expected_recv=261292 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262571 expected_overflow=0 exact_recv_per_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262855 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=163840 expected_recv=262571 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262355 expected_overflow=0 exact_recv_per_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262355 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262196 expected_overflow=0 exact_recv_per_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262683 expected_overflow=0 exact_recv_per_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262323 expected_overflow=0 exact_recv_per_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262559 expected_overflow=0 exact_recv_per_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262196 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=163840 expected_recv=261323 expected_overflow=0 exact_recv_per_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=163840 expected_recv=261532 expected_overflow=0 exact_recv_per_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262683 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262559 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262323 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=163840 expected_recv=261323 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262181 expected_overflow=0 exact_recv_per_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=163840 expected_recv=261532 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=163840 expected_recv=262181 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262534 expected_overflow=0 exact_recv_per_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262534 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262575 expected_overflow=0 exact_recv_per_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262448 expected_overflow=0 exact_recv_per_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262387 expected_overflow=0 exact_recv_per_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262226 expected_overflow=0 exact_recv_per_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262254 expected_overflow=0 exact_recv_per_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262575 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=163840 expected_recv=261520 expected_overflow=0 exact_recv_per_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262448 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=163840 expected_recv=261208 expected_overflow=0 exact_recv_per_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262226 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262387 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=163840 expected_recv=262254 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=163840 expected_recv=261520 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=163840 expected_recv=261208 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=163840 expected_recv=261516 expected_overflow=0 exact_recv_per_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=163840 expected_recv=261516 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=163840 expected_recv=261560 expected_overflow=0 exact_recv_per_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262199 expected_overflow=0 exact_recv_per_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262611 expected_overflow=0 exact_recv_per_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262135 expected_overflow=0 exact_recv_per_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=163840 expected_recv=261560 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262199 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262611 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262736 expected_overflow=0 exact_recv_per_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262135 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262278 expected_overflow=0 exact_recv_per_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262117 expected_overflow=0 exact_recv_per_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262736 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262278 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=163840 expected_recv=262117 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262602 expected_overflow=0 exact_recv_per_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262602 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=163840 expected_recv=261456 expected_overflow=0 exact_recv_per_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262071 expected_overflow=0 exact_recv_per_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=163840 expected_recv=261456 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262362 expected_overflow=0 exact_recv_per_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=163840 expected_recv=261341 expected_overflow=0 exact_recv_per_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262071 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262362 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=163840 expected_recv=261341 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262746 expected_overflow=0 exact_recv_per_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262700 expected_overflow=0 exact_recv_per_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=163840 expected_recv=261874 expected_overflow=0 exact_recv_per_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=163840 expected_recv=263244 expected_overflow=0 exact_recv_per_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262746 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=163840 expected_recv=262700 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=163840 expected_recv=261874 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=163840 expected_recv=263244 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=163840 expected_recv=261374 expected_overflow=0 exact_recv_per_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=163840 expected_recv=261374 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=163840 expected_recv=261339 expected_overflow=0 exact_recv_per_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=163840 expected_recv=262125 expected_overflow=0 exact_recv_per_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=163840 expected_recv=262323 expected_overflow=0 exact_recv_per_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=163840 expected_recv=261339 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=163840 expected_recv=262125 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=163840 expected_recv=262323 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=163840 expected_recv=262727 expected_overflow=0 exact_recv_per_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=163840 expected_recv=262178 expected_overflow=0 exact_recv_per_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=163840 expected_recv=261842 expected_overflow=0 exact_recv_per_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262609 expected_overflow=0 exact_recv_per_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=163840 expected_recv=262727 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=163840 expected_recv=262178 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=163840 expected_recv=261842 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262609 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262483 expected_overflow=0 exact_recv_per_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=163840 expected_recv=261781 expected_overflow=0 exact_recv_per_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=163840 expected_recv=261442 expected_overflow=0 exact_recv_per_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262483 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262996 expected_overflow=0 exact_recv_per_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=163840 expected_recv=261781 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=163840 expected_recv=261442 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=163840 expected_recv=261259 expected_overflow=0 exact_recv_per_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262424 expected_overflow=0 exact_recv_per_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262996 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=163840 expected_recv=261259 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262424 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262158 expected_overflow=0 exact_recv_per_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=163840 expected_recv=261570 expected_overflow=0 exact_recv_per_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=163840 expected_recv=262158 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=163840 expected_recv=261570 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262018 expected_overflow=0 exact_recv_per_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262333 expected_overflow=0 exact_recv_per_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262445 expected_overflow=0 exact_recv_per_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262018 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262812 expected_overflow=0 exact_recv_per_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262333 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262445 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262184 expected_overflow=0 exact_recv_per_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262068 expected_overflow=0 exact_recv_per_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262812 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262184 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=163840 expected_recv=262068 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=163840 expected_recv=261722 expected_overflow=0 exact_recv_per_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=163840 expected_recv=261531 expected_overflow=0 exact_recv_per_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=163840 expected_recv=261722 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=163840 expected_recv=261531 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=163840 expected_recv=261891 expected_overflow=0 exact_recv_per_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=163840 expected_recv=262181 expected_overflow=0 exact_recv_per_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=163840 expected_recv=261891 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=163840 expected_recv=261896 expected_overflow=0 exact_recv_per_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=163840 expected_recv=262181 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=163840 expected_recv=262966 expected_overflow=0 exact_recv_per_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=163840 expected_recv=261896 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=163840 expected_recv=262518 expected_overflow=0 exact_recv_per_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=163840 expected_recv=262050 expected_overflow=0 exact_recv_per_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=163840 expected_recv=262966 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=163840 expected_recv=262518 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=163840 expected_recv=262050 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=163840 expected_recv=262284 expected_overflow=0 exact_recv_per_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=163840 expected_recv=262119 expected_overflow=0 exact_recv_per_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=163840 expected_recv=262284 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=163840 expected_recv=262119 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261695 expected_overflow=0 exact_recv_per_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261976 expected_overflow=0 exact_recv_per_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261695 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=163840 expected_recv=262277 expected_overflow=0 exact_recv_per_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261976 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=163840 expected_recv=262277 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261742 expected_overflow=0 exact_recv_per_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261969 expected_overflow=0 exact_recv_per_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=163840 expected_recv=262169 expected_overflow=0 exact_recv_per_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261742 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=163840 expected_recv=261969 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=163840 expected_recv=262169 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261803 expected_overflow=0 exact_recv_per_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=163840 expected_recv=263040 expected_overflow=0 exact_recv_per_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261803 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261944 expected_overflow=0 exact_recv_per_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=163840 expected_recv=263040 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261824 expected_overflow=0 exact_recv_per_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261944 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261797 expected_overflow=0 exact_recv_per_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261824 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261797 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261608 expected_overflow=0 exact_recv_per_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=163840 expected_recv=262821 expected_overflow=0 exact_recv_per_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=163840 expected_recv=262460 expected_overflow=0 exact_recv_per_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=163840 expected_recv=261608 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=163840 expected_recv=262821 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=163840 expected_recv=262460 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=163840 expected_recv=261499 expected_overflow=0 exact_recv_per_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=163840 expected_recv=262895 expected_overflow=0 exact_recv_per_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=163840 expected_recv=261499 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262431 expected_overflow=0 exact_recv_per_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=163840 expected_recv=262895 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262319 expected_overflow=0 exact_recv_per_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262431 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262630 expected_overflow=0 exact_recv_per_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262319 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262630 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=163840 expected_recv=261906 expected_overflow=0 exact_recv_per_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262159 expected_overflow=0 exact_recv_per_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262038 expected_overflow=0 exact_recv_per_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=163840 expected_recv=261906 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262159 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=163840 expected_recv=262393 expected_overflow=0 exact_recv_per_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262038 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262170 expected_overflow=0 exact_recv_per_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=163840 expected_recv=262393 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=163840 expected_recv=261766 expected_overflow=0 exact_recv_per_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=163840 expected_recv=262170 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=163840 expected_recv=261766 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=163840 expected_recv=261906 expected_overflow=0 exact_recv_per_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=163840 expected_recv=263012 expected_overflow=0 exact_recv_per_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=163840 expected_recv=261906 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=163840 expected_recv=262056 expected_overflow=0 exact_recv_per_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=163840 expected_recv=263012 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=163840 expected_recv=261670 expected_overflow=0 exact_recv_per_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=163840 expected_recv=261944 expected_overflow=0 exact_recv_per_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=163840 expected_recv=262056 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=163840 expected_recv=261670 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=163840 expected_recv=261944 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=163840 expected_recv=262405 expected_overflow=0 exact_recv_per_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=163840 expected_recv=262405 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[131019, 130688, 131271, 130918, 131504, 131362, 130919, 130895] avg=131072.0000 max=131504 max_rank=4 min=130688 min_rank=1 imbalance_ratio=1.003296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[131096, 131178, 130848, 131128, 131445, 131135, 130605, 131141] avg=131072.0000 max=131445 max_rank=4 min=130605 min_rank=6 imbalance_ratio=1.002846 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[131205, 130867, 130996, 131075, 131478, 131274, 130658, 131023] avg=131072.0000 max=131478 max_rank=4 min=130658 min_rank=6 imbalance_ratio=1.003098 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[130523, 131064, 131301, 130990, 131372, 130943, 130892, 131491] avg=131072.0000 max=131491 max_rank=7 min=130523 min_rank=0 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[131120, 131216, 131215, 131440, 130920, 130895, 130660, 131110] avg=131072.0000 max=131440 max_rank=3 min=130660 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[131475, 131349, 131231, 131084, 130207, 131048, 131273, 130909] avg=131072.0000 max=131475 max_rank=0 min=130207 min_rank=4 imbalance_ratio=1.003075 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[130974, 131576, 130958, 131100, 130802, 131173, 131455, 130538] avg=131072.0000 max=131576 max_rank=1 min=130538 min_rank=7 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[131156, 131567, 130504, 130595, 130814, 131300, 131448, 131192] avg=131072.0000 max=131567 max_rank=1 min=130504 min_rank=2 imbalance_ratio=1.003777 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[131496, 130730, 131523, 130408, 130802, 131079, 131408, 131130] avg=131072.0000 max=131523 max_rank=2 min=130408 min_rank=3 imbalance_ratio=1.003441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[131286, 130719, 131444, 131267, 131103, 131145, 130371, 131241] avg=131072.0000 max=131444 max_rank=2 min=130371 min_rank=6 imbalance_ratio=1.002838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[130796, 131449, 131519, 131091, 131137, 130493, 131010, 131081] avg=131072.0000 max=131519 max_rank=2 min=130493 min_rank=5 imbalance_ratio=1.003410 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[130522, 131040, 131623, 130893, 131048, 131129, 131336, 130985] avg=131072.0000 max=131623 max_rank=2 min=130522 min_rank=0 imbalance_ratio=1.004204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[131276, 130800, 130738, 130769, 131161, 131330, 131102, 131400] avg=131072.0000 max=131400 max_rank=7 min=130738 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[130700, 130669, 130779, 131155, 131276, 131376, 131543, 131078] avg=131072.0000 max=131543 max_rank=6 min=130669 min_rank=1 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[130631, 131175, 130846, 131134, 130676, 131176, 131210, 131728] avg=131072.0000 max=131728 max_rank=7 min=130631 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[131447, 131542, 130961, 130496, 131089, 131329, 130611, 131101] avg=131072.0000 max=131542 max_rank=1 min=130496 min_rank=3 imbalance_ratio=1.003586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=4193111 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016832 peak_demand=131491 required_capacity=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=4192804 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017728 peak_demand=131728 required_capacity=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=4194490 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016832 peak_demand=131504 required_capacity=1.003296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=4193894 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017088 peak_demand=131748 required_capacity=1.005157 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=4194694 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015040 peak_demand=131576 required_capacity=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=4195815 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014144 peak_demand=131623 required_capacity=1.004204 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=4194313 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016448 peak_demand=131963 required_capacity=1.006798 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=4195311 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014656 peak_demand=131710 required_capacity=1.004868 + [2026-08-30 05:20:23] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 548.3 | throughput per GPU (TFLOP/s/GPU): 608.0 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.034104E+01 | loss scale: 1.0 | grad norm: 17.232 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 132864 | min safe capacity: 1.0137 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=163840 expected_recv=262015 expected_overflow=0 exact_recv_per_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=163840 expected_recv=261290 expected_overflow=0 exact_recv_per_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=163840 expected_recv=261997 expected_overflow=0 exact_recv_per_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=163840 expected_recv=262093 expected_overflow=0 exact_recv_per_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=163840 expected_recv=262622 expected_overflow=0 exact_recv_per_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=163840 expected_recv=262561 expected_overflow=0 exact_recv_per_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=163840 expected_recv=262595 expected_overflow=0 exact_recv_per_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=163840 expected_recv=262015 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=163840 expected_recv=261290 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=163840 expected_recv=261979 expected_overflow=0 exact_recv_per_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=163840 expected_recv=261997 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=163840 expected_recv=262093 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=163840 expected_recv=262622 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=163840 expected_recv=262561 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=163840 expected_recv=262595 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=163840 expected_recv=261979 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=163840 expected_recv=262050 expected_overflow=0 exact_recv_per_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=163840 expected_recv=261915 expected_overflow=0 exact_recv_per_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=163840 expected_recv=262050 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=163840 expected_recv=261915 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=163840 expected_recv=262520 expected_overflow=0 exact_recv_per_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=163840 expected_recv=261498 expected_overflow=0 exact_recv_per_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=163840 expected_recv=261562 expected_overflow=0 exact_recv_per_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=163840 expected_recv=262348 expected_overflow=0 exact_recv_per_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=163840 expected_recv=262556 expected_overflow=0 exact_recv_per_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=163840 expected_recv=262520 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=163840 expected_recv=261498 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=163840 expected_recv=261562 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=163840 expected_recv=262703 expected_overflow=0 exact_recv_per_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=163840 expected_recv=262348 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=163840 expected_recv=262556 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=163840 expected_recv=262703 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262373 expected_overflow=0 exact_recv_per_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262743 expected_overflow=0 exact_recv_per_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262373 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262743 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262804 expected_overflow=0 exact_recv_per_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=163840 expected_recv=261968 expected_overflow=0 exact_recv_per_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=163840 expected_recv=261330 expected_overflow=0 exact_recv_per_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262428 expected_overflow=0 exact_recv_per_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262804 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=163840 expected_recv=261968 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=163840 expected_recv=261797 expected_overflow=0 exact_recv_per_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=163840 expected_recv=261330 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=163840 expected_recv=261709 expected_overflow=0 exact_recv_per_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=163840 expected_recv=262428 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=163840 expected_recv=261797 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=163840 expected_recv=261709 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262537 expected_overflow=0 exact_recv_per_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=163840 expected_recv=261986 expected_overflow=0 exact_recv_per_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262537 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=163840 expected_recv=261986 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262309 expected_overflow=0 exact_recv_per_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262169 expected_overflow=0 exact_recv_per_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262150 expected_overflow=0 exact_recv_per_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262313 expected_overflow=0 exact_recv_per_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262309 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262169 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262150 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262313 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262344 expected_overflow=0 exact_recv_per_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=163840 expected_recv=261344 expected_overflow=0 exact_recv_per_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=163840 expected_recv=262344 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=163840 expected_recv=261344 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=163840 expected_recv=261577 expected_overflow=0 exact_recv_per_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=163840 expected_recv=261916 expected_overflow=0 exact_recv_per_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=163840 expected_recv=261577 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=163840 expected_recv=261916 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=163840 expected_recv=262489 expected_overflow=0 exact_recv_per_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=163840 expected_recv=262833 expected_overflow=0 exact_recv_per_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=163840 expected_recv=261989 expected_overflow=0 exact_recv_per_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=163840 expected_recv=261944 expected_overflow=0 exact_recv_per_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=163840 expected_recv=262489 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=163840 expected_recv=262833 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=163840 expected_recv=261989 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=163840 expected_recv=261944 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=163840 expected_recv=262092 expected_overflow=0 exact_recv_per_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=163840 expected_recv=262312 expected_overflow=0 exact_recv_per_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=163840 expected_recv=262092 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=163840 expected_recv=262312 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=163840 expected_recv=262335 expected_overflow=0 exact_recv_per_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=163840 expected_recv=261827 expected_overflow=0 exact_recv_per_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=163840 expected_recv=262335 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=163840 expected_recv=261827 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=163840 expected_recv=262202 expected_overflow=0 exact_recv_per_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=163840 expected_recv=262420 expected_overflow=0 exact_recv_per_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=163840 expected_recv=262768 expected_overflow=0 exact_recv_per_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=163840 expected_recv=261724 expected_overflow=0 exact_recv_per_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=163840 expected_recv=262202 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=163840 expected_recv=262420 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=163840 expected_recv=262768 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=163840 expected_recv=261818 expected_overflow=0 exact_recv_per_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=163840 expected_recv=261724 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=163840 expected_recv=262058 expected_overflow=0 exact_recv_per_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=163840 expected_recv=261818 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=163840 expected_recv=262575 expected_overflow=0 exact_recv_per_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=163840 expected_recv=262058 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=163840 expected_recv=262406 expected_overflow=0 exact_recv_per_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=163840 expected_recv=262575 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=163840 expected_recv=262406 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=163840 expected_recv=262241 expected_overflow=0 exact_recv_per_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=163840 expected_recv=261450 expected_overflow=0 exact_recv_per_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=163840 expected_recv=262346 expected_overflow=0 exact_recv_per_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=163840 expected_recv=262489 expected_overflow=0 exact_recv_per_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=163840 expected_recv=262241 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=163840 expected_recv=261450 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=163840 expected_recv=262346 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=163840 expected_recv=261961 expected_overflow=0 exact_recv_per_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=163840 expected_recv=262489 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=163840 expected_recv=261684 expected_overflow=0 exact_recv_per_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=163840 expected_recv=261961 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262031 expected_overflow=0 exact_recv_per_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262094 expected_overflow=0 exact_recv_per_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=163840 expected_recv=261684 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262031 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262094 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262217 expected_overflow=0 exact_recv_per_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262017 expected_overflow=0 exact_recv_per_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262217 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262574 expected_overflow=0 exact_recv_per_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=163840 expected_recv=261681 expected_overflow=0 exact_recv_per_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262017 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262240 expected_overflow=0 exact_recv_per_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262574 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=163840 expected_recv=261681 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262240 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262298 expected_overflow=0 exact_recv_per_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262170 expected_overflow=0 exact_recv_per_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=163840 expected_recv=261374 expected_overflow=0 exact_recv_per_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=163840 expected_recv=262298 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262170 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=163840 expected_recv=261374 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=163840 expected_recv=261417 expected_overflow=0 exact_recv_per_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262329 expected_overflow=0 exact_recv_per_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=163840 expected_recv=261417 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262329 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262003 expected_overflow=0 exact_recv_per_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262581 expected_overflow=0 exact_recv_per_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262502 expected_overflow=0 exact_recv_per_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262003 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262776 expected_overflow=0 exact_recv_per_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=163840 expected_recv=262409 expected_overflow=0 exact_recv_per_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=163840 expected_recv=261406 expected_overflow=0 exact_recv_per_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262581 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262502 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=163840 expected_recv=262776 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=163840 expected_recv=262409 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=163840 expected_recv=261406 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=163840 expected_recv=261936 expected_overflow=0 exact_recv_per_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=163840 expected_recv=262503 expected_overflow=0 exact_recv_per_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=163840 expected_recv=261936 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=163840 expected_recv=262503 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=163840 expected_recv=261866 expected_overflow=0 exact_recv_per_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=163840 expected_recv=262389 expected_overflow=0 exact_recv_per_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=163840 expected_recv=261866 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=163840 expected_recv=262585 expected_overflow=0 exact_recv_per_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=163840 expected_recv=262058 expected_overflow=0 exact_recv_per_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=163840 expected_recv=262324 expected_overflow=0 exact_recv_per_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=163840 expected_recv=262389 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=163840 expected_recv=261240 expected_overflow=0 exact_recv_per_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=163840 expected_recv=262585 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=163840 expected_recv=262058 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=163840 expected_recv=262324 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=163840 expected_recv=261240 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=163840 expected_recv=261225 expected_overflow=0 exact_recv_per_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=163840 expected_recv=262937 expected_overflow=0 exact_recv_per_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=163840 expected_recv=261225 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=163840 expected_recv=262937 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=163840 expected_recv=263093 expected_overflow=0 exact_recv_per_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=163840 expected_recv=262146 expected_overflow=0 exact_recv_per_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=163840 expected_recv=263093 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=163840 expected_recv=261635 expected_overflow=0 exact_recv_per_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=163840 expected_recv=262146 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=163840 expected_recv=262552 expected_overflow=0 exact_recv_per_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262211 expected_overflow=0 exact_recv_per_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=163840 expected_recv=261635 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262956 expected_overflow=0 exact_recv_per_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=163840 expected_recv=262552 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262211 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262205 expected_overflow=0 exact_recv_per_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262956 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262522 expected_overflow=0 exact_recv_per_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262205 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262522 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=163840 expected_recv=261055 expected_overflow=0 exact_recv_per_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=163840 expected_recv=261954 expected_overflow=0 exact_recv_per_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=163840 expected_recv=261055 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262295 expected_overflow=0 exact_recv_per_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=163840 expected_recv=261954 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=163840 expected_recv=261954 expected_overflow=0 exact_recv_per_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262213 expected_overflow=0 exact_recv_per_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=163840 expected_recv=262295 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=163840 expected_recv=261810 expected_overflow=0 exact_recv_per_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=163840 expected_recv=261326 expected_overflow=0 exact_recv_per_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=163840 expected_recv=261954 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262213 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262043 expected_overflow=0 exact_recv_per_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=163840 expected_recv=261810 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=163840 expected_recv=261326 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262043 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262466 expected_overflow=0 exact_recv_per_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=163840 expected_recv=261638 expected_overflow=0 exact_recv_per_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262466 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262690 expected_overflow=0 exact_recv_per_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=163840 expected_recv=261638 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262966 expected_overflow=0 exact_recv_per_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262401 expected_overflow=0 exact_recv_per_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=163840 expected_recv=261366 expected_overflow=0 exact_recv_per_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262690 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=163840 expected_recv=261914 expected_overflow=0 exact_recv_per_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=163840 expected_recv=262966 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262401 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262425 expected_overflow=0 exact_recv_per_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=163840 expected_recv=261366 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=163840 expected_recv=261914 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262425 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262562 expected_overflow=0 exact_recv_per_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=163840 expected_recv=261663 expected_overflow=0 exact_recv_per_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262562 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=163840 expected_recv=261663 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262336 expected_overflow=0 exact_recv_per_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262029 expected_overflow=0 exact_recv_per_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262485 expected_overflow=0 exact_recv_per_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262336 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262186 expected_overflow=0 exact_recv_per_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262213 expected_overflow=0 exact_recv_per_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262029 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262353 expected_overflow=0 exact_recv_per_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262186 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=163840 expected_recv=262485 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262213 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262353 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262499 expected_overflow=0 exact_recv_per_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=163840 expected_recv=261802 expected_overflow=0 exact_recv_per_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262499 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=163840 expected_recv=261802 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262244 expected_overflow=0 exact_recv_per_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=163840 expected_recv=261630 expected_overflow=0 exact_recv_per_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262756 expected_overflow=0 exact_recv_per_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262518 expected_overflow=0 exact_recv_per_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=163840 expected_recv=262244 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=163840 expected_recv=261826 expected_overflow=0 exact_recv_per_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262424 expected_overflow=0 exact_recv_per_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=163840 expected_recv=261630 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262756 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262518 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=163840 expected_recv=261826 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262135 expected_overflow=0 exact_recv_per_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262424 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=163840 expected_recv=261518 expected_overflow=0 exact_recv_per_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262135 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=163840 expected_recv=261518 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262569 expected_overflow=0 exact_recv_per_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=163840 expected_recv=262569 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=163840 expected_recv=261602 expected_overflow=0 exact_recv_per_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=163840 expected_recv=261602 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[130970, 131041, 130796, 130988, 131086, 131011, 131284, 131400] avg=131072.0000 max=131400 max_rank=7 min=130796 min_rank=2 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[131168, 130530, 131038, 131548, 131000, 131244, 130671, 131377] avg=131072.0000 max=131548 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[130685, 130530, 131480, 131749, 131319, 130650, 130828, 131335] avg=131072.0000 max=131749 max_rank=3 min=130530 min_rank=1 imbalance_ratio=1.005165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[131320, 130886, 130916, 131457, 131178, 130527, 131089, 131203] avg=131072.0000 max=131457 max_rank=3 min=130527 min_rank=5 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[130934, 131426, 130816, 130306, 131084, 131499, 130915, 131596] avg=131072.0000 max=131596 max_rank=7 min=130306 min_rank=3 imbalance_ratio=1.003998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[131074, 130865, 131060, 131015, 131429, 130693, 131048, 131392] avg=131072.0000 max=131429 max_rank=4 min=130693 min_rank=5 imbalance_ratio=1.002724 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[131273, 130345, 131251, 131457, 131149, 130843, 131278, 130980] avg=131072.0000 max=131457 max_rank=3 min=130345 min_rank=1 imbalance_ratio=1.002937 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[130850, 131215, 130991, 130949, 131040, 130949, 131433, 131149] avg=131072.0000 max=131433 max_rank=6 min=130850 min_rank=0 imbalance_ratio=1.002754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[130732, 131065, 130998, 131018, 131356, 131556, 131217, 130634] avg=131072.0000 max=131556 max_rank=5 min=130634 min_rank=7 imbalance_ratio=1.003693 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[130530, 131043, 131402, 131311, 131398, 130916, 131061, 130915] avg=131072.0000 max=131402 max_rank=2 min=130530 min_rank=0 imbalance_ratio=1.002518 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[130405, 131411, 131178, 130953, 131244, 131304, 131388, 130693] avg=131072.0000 max=131411 max_rank=1 min=130405 min_rank=0 imbalance_ratio=1.002586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[131262, 131140, 131211, 131220, 131297, 130774, 130645, 131027] avg=131072.0000 max=131297 max_rank=4 min=130645 min_rank=6 imbalance_ratio=1.001717 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[131099, 130776, 131167, 131832, 130663, 131359, 131041, 130639] avg=131072.0000 max=131832 max_rank=3 min=130639 min_rank=7 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[130797, 131237, 131250, 130859, 130767, 131234, 131333, 131099] avg=131072.0000 max=131333 max_rank=6 min=130767 min_rank=4 imbalance_ratio=1.001991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[131083, 131136, 130992, 131202, 130907, 131009, 131097, 131150] avg=131072.0000 max=131202 max_rank=3 min=130907 min_rank=4 imbalance_ratio=1.000992 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[131054, 131101, 130557, 131237, 130810, 131044, 131214, 131559] avg=131072.0000 max=131559 max_rank=7 min=130557 min_rank=2 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=4194190 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016448 peak_demand=131596 required_capacity=1.003998 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=4195422 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015936 peak_demand=131526 required_capacity=1.003464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=4192537 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1019008 peak_demand=131703 required_capacity=1.004814 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=4192074 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018752 peak_demand=131694 required_capacity=1.004745 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=4193839 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017472 peak_demand=131480 required_capacity=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=4195183 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016064 peak_demand=131720 required_capacity=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=4196324 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1011584 peak_demand=131832 required_capacity=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=4194863 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016320 peak_demand=131607 required_capacity=1.004082 + [2026-08-30 05:20:24] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 548.1 | throughput per GPU (TFLOP/s/GPU): 608.3 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.016406E+01 | loss scale: 1.0 | grad norm: 17.538 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 132864 | min safe capacity: 1.0137 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262011 expected_overflow=0 exact_recv_per_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=163840 expected_recv=261458 expected_overflow=0 exact_recv_per_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262690 expected_overflow=0 exact_recv_per_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262302 expected_overflow=0 exact_recv_per_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=163840 expected_recv=261689 expected_overflow=0 exact_recv_per_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262117 expected_overflow=0 exact_recv_per_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262437 expected_overflow=0 exact_recv_per_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262011 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=163840 expected_recv=261458 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262690 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262448 expected_overflow=0 exact_recv_per_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262302 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=163840 expected_recv=261689 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262117 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262437 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=163840 expected_recv=262448 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=163840 expected_recv=262834 expected_overflow=0 exact_recv_per_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=163840 expected_recv=261641 expected_overflow=0 exact_recv_per_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=163840 expected_recv=261561 expected_overflow=0 exact_recv_per_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=163840 expected_recv=262185 expected_overflow=0 exact_recv_per_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=163840 expected_recv=262104 expected_overflow=0 exact_recv_per_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=163840 expected_recv=262834 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=163840 expected_recv=263583 expected_overflow=0 exact_recv_per_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=163840 expected_recv=261641 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=163840 expected_recv=261843 expected_overflow=0 exact_recv_per_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=163840 expected_recv=261561 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=163840 expected_recv=262185 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=163840 expected_recv=262104 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=163840 expected_recv=261401 expected_overflow=0 exact_recv_per_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=163840 expected_recv=263583 actual_valid_recv=265728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=163840 expected_recv=261843 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=265728 slack_slots=61952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=163840 expected_recv=261401 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262274 expected_overflow=0 exact_recv_per_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=163840 expected_recv=261880 expected_overflow=0 exact_recv_per_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=163840 expected_recv=261997 expected_overflow=0 exact_recv_per_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262274 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262307 expected_overflow=0 exact_recv_per_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262439 expected_overflow=0 exact_recv_per_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262430 expected_overflow=0 exact_recv_per_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=163840 expected_recv=261880 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=163840 expected_recv=261997 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262307 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262439 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262430 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262571 expected_overflow=0 exact_recv_per_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=163840 expected_recv=261254 expected_overflow=0 exact_recv_per_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=163840 expected_recv=262571 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=163840 expected_recv=261254 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=163840 expected_recv=261939 expected_overflow=0 exact_recv_per_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=163840 expected_recv=262584 expected_overflow=0 exact_recv_per_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=163840 expected_recv=261939 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=163840 expected_recv=261486 expected_overflow=0 exact_recv_per_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=163840 expected_recv=261969 expected_overflow=0 exact_recv_per_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=163840 expected_recv=261867 expected_overflow=0 exact_recv_per_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=163840 expected_recv=262598 expected_overflow=0 exact_recv_per_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=163840 expected_recv=262584 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=163840 expected_recv=261486 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=163840 expected_recv=261969 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=163840 expected_recv=261867 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=163840 expected_recv=262598 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=163840 expected_recv=262250 expected_overflow=0 exact_recv_per_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=163840 expected_recv=262459 expected_overflow=0 exact_recv_per_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=163840 expected_recv=262250 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=163840 expected_recv=262459 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262039 expected_overflow=0 exact_recv_per_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262039 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262228 expected_overflow=0 exact_recv_per_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=163840 expected_recv=261921 expected_overflow=0 exact_recv_per_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262796 expected_overflow=0 exact_recv_per_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262071 expected_overflow=0 exact_recv_per_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=163840 expected_recv=261567 expected_overflow=0 exact_recv_per_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262228 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=163840 expected_recv=261921 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262796 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=163840 expected_recv=261867 expected_overflow=0 exact_recv_per_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262071 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262663 expected_overflow=0 exact_recv_per_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=163840 expected_recv=261567 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=163840 expected_recv=261867 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=163840 expected_recv=262663 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=163840 expected_recv=261665 expected_overflow=0 exact_recv_per_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=163840 expected_recv=261665 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=163840 expected_recv=262072 expected_overflow=0 exact_recv_per_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=163840 expected_recv=261866 expected_overflow=0 exact_recv_per_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=163840 expected_recv=262393 expected_overflow=0 exact_recv_per_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=163840 expected_recv=262654 expected_overflow=0 exact_recv_per_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=163840 expected_recv=262072 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=163840 expected_recv=262543 expected_overflow=0 exact_recv_per_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=163840 expected_recv=262733 expected_overflow=0 exact_recv_per_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=163840 expected_recv=261866 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=163840 expected_recv=261226 expected_overflow=0 exact_recv_per_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=163840 expected_recv=262393 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=163840 expected_recv=262654 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=163840 expected_recv=262543 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=163840 expected_recv=262733 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=163840 expected_recv=261226 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262035 expected_overflow=0 exact_recv_per_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262035 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262732 expected_overflow=0 exact_recv_per_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262323 expected_overflow=0 exact_recv_per_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262732 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262323 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=163840 expected_recv=261759 expected_overflow=0 exact_recv_per_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=163840 expected_recv=261947 expected_overflow=0 exact_recv_per_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=163840 expected_recv=261180 expected_overflow=0 exact_recv_per_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262566 expected_overflow=0 exact_recv_per_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=163840 expected_recv=261759 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262610 expected_overflow=0 exact_recv_per_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=163840 expected_recv=261947 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=163840 expected_recv=261180 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262566 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=163840 expected_recv=262610 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=163840 expected_recv=262793 expected_overflow=0 exact_recv_per_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=163840 expected_recv=262793 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=163840 expected_recv=262618 expected_overflow=0 exact_recv_per_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=163840 expected_recv=262090 expected_overflow=0 exact_recv_per_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=163840 expected_recv=262618 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=163840 expected_recv=262090 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=163840 expected_recv=261775 expected_overflow=0 exact_recv_per_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=163840 expected_recv=261927 expected_overflow=0 exact_recv_per_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=163840 expected_recv=261982 expected_overflow=0 exact_recv_per_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=163840 expected_recv=262243 expected_overflow=0 exact_recv_per_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=163840 expected_recv=261724 expected_overflow=0 exact_recv_per_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=163840 expected_recv=261775 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=163840 expected_recv=261927 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=163840 expected_recv=261982 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=163840 expected_recv=262243 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=163840 expected_recv=261724 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=163840 expected_recv=261764 expected_overflow=0 exact_recv_per_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=163840 expected_recv=261764 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=163840 expected_recv=261902 expected_overflow=0 exact_recv_per_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262731 expected_overflow=0 exact_recv_per_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=163840 expected_recv=261902 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262731 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262204 expected_overflow=0 exact_recv_per_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262226 expected_overflow=0 exact_recv_per_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262043 expected_overflow=0 exact_recv_per_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262043 expected_overflow=0 exact_recv_per_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262239 expected_overflow=0 exact_recv_per_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262204 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262226 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262043 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262043 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=163840 expected_recv=262239 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262661 expected_overflow=0 exact_recv_per_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=163840 expected_recv=261676 expected_overflow=0 exact_recv_per_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262661 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=163840 expected_recv=261676 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=163840 expected_recv=261318 expected_overflow=0 exact_recv_per_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=163840 expected_recv=261318 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=163840 expected_recv=261829 expected_overflow=0 exact_recv_per_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262943 expected_overflow=0 exact_recv_per_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=163840 expected_recv=261829 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262499 expected_overflow=0 exact_recv_per_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262474 expected_overflow=0 exact_recv_per_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=163840 expected_recv=261752 expected_overflow=0 exact_recv_per_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262943 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262499 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=163840 expected_recv=262474 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=163840 expected_recv=261752 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=163840 expected_recv=261946 expected_overflow=0 exact_recv_per_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=163840 expected_recv=261643 expected_overflow=0 exact_recv_per_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=163840 expected_recv=261946 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=163840 expected_recv=261643 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=163840 expected_recv=262424 expected_overflow=0 exact_recv_per_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=163840 expected_recv=262424 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=163840 expected_recv=261949 expected_overflow=0 exact_recv_per_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=163840 expected_recv=261949 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=163840 expected_recv=262639 expected_overflow=0 exact_recv_per_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=163840 expected_recv=261996 expected_overflow=0 exact_recv_per_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=163840 expected_recv=262116 expected_overflow=0 exact_recv_per_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=163840 expected_recv=262439 expected_overflow=0 exact_recv_per_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=163840 expected_recv=262639 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=163840 expected_recv=261996 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=163840 expected_recv=262346 expected_overflow=0 exact_recv_per_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=163840 expected_recv=262116 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=163840 expected_recv=262439 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=163840 expected_recv=262756 expected_overflow=0 exact_recv_per_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=163840 expected_recv=262346 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=163840 expected_recv=261847 expected_overflow=0 exact_recv_per_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=163840 expected_recv=262756 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=163840 expected_recv=262185 expected_overflow=0 exact_recv_per_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=163840 expected_recv=261847 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=163840 expected_recv=262185 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=163840 expected_recv=262031 expected_overflow=0 exact_recv_per_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=163840 expected_recv=262768 expected_overflow=0 exact_recv_per_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=163840 expected_recv=261559 expected_overflow=0 exact_recv_per_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=163840 expected_recv=261660 expected_overflow=0 exact_recv_per_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=163840 expected_recv=262031 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262470 expected_overflow=0 exact_recv_per_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=163840 expected_recv=262768 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=163840 expected_recv=261559 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=163840 expected_recv=261660 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262470 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262464 expected_overflow=0 exact_recv_per_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262859 expected_overflow=0 exact_recv_per_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262807 expected_overflow=0 exact_recv_per_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262464 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262859 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262807 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=163840 expected_recv=261820 expected_overflow=0 exact_recv_per_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=163840 expected_recv=261623 expected_overflow=0 exact_recv_per_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=163840 expected_recv=261820 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=163840 expected_recv=261623 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262025 expected_overflow=0 exact_recv_per_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=163840 expected_recv=261084 expected_overflow=0 exact_recv_per_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262297 expected_overflow=0 exact_recv_per_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=163840 expected_recv=262025 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=163840 expected_recv=261084 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262168 expected_overflow=0 exact_recv_per_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262297 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=163840 expected_recv=261527 expected_overflow=0 exact_recv_per_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=163840 expected_recv=261935 expected_overflow=0 exact_recv_per_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262168 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=163840 expected_recv=261527 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=163840 expected_recv=261935 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262783 expected_overflow=0 exact_recv_per_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262050 expected_overflow=0 exact_recv_per_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262783 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262043 expected_overflow=0 exact_recv_per_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262349 expected_overflow=0 exact_recv_per_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=163840 expected_recv=261992 expected_overflow=0 exact_recv_per_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262050 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262043 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262915 expected_overflow=0 exact_recv_per_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=163840 expected_recv=261992 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=163840 expected_recv=262349 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262545 expected_overflow=0 exact_recv_per_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=163840 expected_recv=261617 expected_overflow=0 exact_recv_per_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262915 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262545 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=163840 expected_recv=261617 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262437 expected_overflow=0 exact_recv_per_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262437 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262103 expected_overflow=0 exact_recv_per_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262174 expected_overflow=0 exact_recv_per_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=163840 expected_recv=261369 expected_overflow=0 exact_recv_per_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=163840 expected_recv=262065 expected_overflow=0 exact_recv_per_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262103 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=163840 expected_recv=262174 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=163840 expected_recv=261369 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=163840 expected_recv=262065 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=163840 expected_recv=261823 expected_overflow=0 exact_recv_per_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=163840 expected_recv=262615 expected_overflow=0 exact_recv_per_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=163840 expected_recv=262566 expected_overflow=0 exact_recv_per_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=163840 expected_recv=261823 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=163840 expected_recv=261668 expected_overflow=0 exact_recv_per_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=163840 expected_recv=262615 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=163840 expected_recv=262566 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=163840 expected_recv=261668 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=163840 expected_recv=262224 expected_overflow=0 exact_recv_per_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=163840 expected_recv=262510 expected_overflow=0 exact_recv_per_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=163840 expected_recv=262224 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=163840 expected_recv=262510 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=163840 expected_recv=261681 expected_overflow=0 exact_recv_per_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=163840 expected_recv=261681 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[130981, 131138, 130597, 131184, 131340, 130868, 131136, 131332] avg=131072.0000 max=131340 max_rank=4 min=130597 min_rank=2 imbalance_ratio=1.002045 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[131500, 130843, 130957, 130985, 130544, 131002, 130980, 131765] avg=131072.0000 max=131765 max_rank=7 min=130544 min_rank=4 imbalance_ratio=1.005287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[130866, 130868, 131264, 130774, 130753, 131411, 131182, 131458] avg=131072.0000 max=131458 max_rank=7 min=130753 min_rank=4 imbalance_ratio=1.002945 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[130846, 130722, 130841, 131458, 131234, 130941, 131051, 131483] avg=131072.0000 max=131483 max_rank=7 min=130722 min_rank=1 imbalance_ratio=1.003136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[130701, 131059, 131288, 131055, 131604, 130783, 131038, 131048] avg=131072.0000 max=131604 max_rank=4 min=130701 min_rank=0 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[130828, 130903, 131344, 130962, 131666, 131226, 130584, 131063] avg=131072.0000 max=131666 max_rank=4 min=130584 min_rank=6 imbalance_ratio=1.004532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[131105, 131154, 130880, 131352, 131070, 130726, 131397, 130892] avg=131072.0000 max=131397 max_rank=6 min=130726 min_rank=5 imbalance_ratio=1.002480 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[131486, 131302, 130654, 131021, 131277, 130885, 130817, 131134] avg=131072.0000 max=131486 max_rank=0 min=130654 min_rank=2 imbalance_ratio=1.003159 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[130924, 131023, 130978, 131485, 131117, 130893, 130689, 131467] avg=131072.0000 max=131485 max_rank=3 min=130689 min_rank=6 imbalance_ratio=1.003151 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[131334, 130598, 131455, 130327, 131510, 131123, 131440, 130789] avg=131072.0000 max=131510 max_rank=4 min=130327 min_rank=3 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[131030, 130773, 130752, 130947, 130817, 131264, 131841, 131152] avg=131072.0000 max=131841 max_rank=6 min=130752 min_rank=2 imbalance_ratio=1.005867 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[131203, 131268, 130855, 131206, 131575, 130862, 130647, 130960] avg=131072.0000 max=131575 max_rank=4 min=130647 min_rank=6 imbalance_ratio=1.003838 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[131029, 131221, 131478, 130882, 130655, 130604, 131006, 131701] avg=131072.0000 max=131701 max_rank=7 min=130604 min_rank=5 imbalance_ratio=1.004799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[131068, 131203, 131002, 130977, 131262, 130880, 131232, 130952] avg=131072.0000 max=131262 max_rank=4 min=130880 min_rank=5 imbalance_ratio=1.001450 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[131175, 131510, 130635, 131366, 130801, 130902, 130965, 131222] avg=131072.0000 max=131510 max_rank=1 min=130635 min_rank=2 imbalance_ratio=1.003342 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[130977, 130843, 130833, 130980, 130900, 130999, 131470, 131574] avg=131072.0000 max=131574 max_rank=7 min=130833 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=4194660 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014528 peak_demand=131578 required_capacity=1.003860 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=4195818 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014784 peak_demand=131818 required_capacity=1.005692 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=4195131 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014784 peak_demand=131500 required_capacity=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=4195351 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014528 peak_demand=131666 required_capacity=1.004532 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=4194892 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015168 peak_demand=131841 required_capacity=1.005867 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=4191272 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1020160 peak_demand=131511 required_capacity=1.003349 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=4195168 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016448 peak_demand=131977 required_capacity=1.006905 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=4192140 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018368 peak_demand=131508 required_capacity=1.003326 + [2026-08-30 05:20:24] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 546.8 | throughput per GPU (TFLOP/s/GPU): 609.7 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.902584E+00 | loss scale: 1.0 | grad norm: 6.121 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 132864 | min safe capacity: 1.0137 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262005 expected_overflow=0 exact_recv_per_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262277 expected_overflow=0 exact_recv_per_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262551 expected_overflow=0 exact_recv_per_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=163840 expected_recv=261409 expected_overflow=0 exact_recv_per_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262766 expected_overflow=0 exact_recv_per_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=163840 expected_recv=261388 expected_overflow=0 exact_recv_per_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262537 expected_overflow=0 exact_recv_per_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262005 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262277 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262219 expected_overflow=0 exact_recv_per_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262551 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=163840 expected_recv=261409 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262766 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=163840 expected_recv=261388 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262537 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=163840 expected_recv=262219 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262238 expected_overflow=0 exact_recv_per_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262252 expected_overflow=0 exact_recv_per_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=163840 expected_recv=261725 expected_overflow=0 exact_recv_per_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262238 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262714 expected_overflow=0 exact_recv_per_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262252 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262071 expected_overflow=0 exact_recv_per_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=163840 expected_recv=261725 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262714 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262071 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262664 expected_overflow=0 exact_recv_per_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=163840 expected_recv=261183 expected_overflow=0 exact_recv_per_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262305 expected_overflow=0 exact_recv_per_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262664 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=163840 expected_recv=261183 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=163840 expected_recv=262305 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=163840 expected_recv=262272 expected_overflow=0 exact_recv_per_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=163840 expected_recv=261781 expected_overflow=0 exact_recv_per_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=163840 expected_recv=262272 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=163840 expected_recv=261781 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=163840 expected_recv=262093 expected_overflow=0 exact_recv_per_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=163840 expected_recv=262924 expected_overflow=0 exact_recv_per_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=163840 expected_recv=261540 expected_overflow=0 exact_recv_per_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=163840 expected_recv=262093 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=163840 expected_recv=262216 expected_overflow=0 exact_recv_per_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=163840 expected_recv=262924 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=163840 expected_recv=261864 expected_overflow=0 exact_recv_per_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=163840 expected_recv=262462 expected_overflow=0 exact_recv_per_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=163840 expected_recv=261540 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=163840 expected_recv=262216 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=163840 expected_recv=262462 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=163840 expected_recv=261864 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=163840 expected_recv=261757 expected_overflow=0 exact_recv_per_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262567 expected_overflow=0 exact_recv_per_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=163840 expected_recv=261757 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262567 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262517 expected_overflow=0 exact_recv_per_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262176 expected_overflow=0 exact_recv_per_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=163840 expected_recv=261625 expected_overflow=0 exact_recv_per_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262517 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262024 expected_overflow=0 exact_recv_per_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262176 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262031 expected_overflow=0 exact_recv_per_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262455 expected_overflow=0 exact_recv_per_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=163840 expected_recv=261625 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262024 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262031 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=163840 expected_recv=262455 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=163840 expected_recv=261504 expected_overflow=0 exact_recv_per_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262298 expected_overflow=0 exact_recv_per_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=163840 expected_recv=261504 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262298 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262723 expected_overflow=0 exact_recv_per_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262556 expected_overflow=0 exact_recv_per_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262723 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262139 expected_overflow=0 exact_recv_per_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262556 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262101 expected_overflow=0 exact_recv_per_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262139 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262512 expected_overflow=0 exact_recv_per_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=163840 expected_recv=261319 expected_overflow=0 exact_recv_per_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262101 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=163840 expected_recv=262512 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=163840 expected_recv=261319 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=163840 expected_recv=262762 expected_overflow=0 exact_recv_per_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261649 expected_overflow=0 exact_recv_per_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=163840 expected_recv=262762 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261649 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261737 expected_overflow=0 exact_recv_per_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=163840 expected_recv=262162 expected_overflow=0 exact_recv_per_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261737 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=163840 expected_recv=262953 expected_overflow=0 exact_recv_per_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=163840 expected_recv=262162 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261526 expected_overflow=0 exact_recv_per_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=163840 expected_recv=262953 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261863 expected_overflow=0 exact_recv_per_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=163840 expected_recv=262500 expected_overflow=0 exact_recv_per_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261526 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262069 expected_overflow=0 exact_recv_per_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=163840 expected_recv=261863 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262699 expected_overflow=0 exact_recv_per_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=163840 expected_recv=262500 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262069 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262699 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=163840 expected_recv=261177 expected_overflow=0 exact_recv_per_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=163840 expected_recv=261834 expected_overflow=0 exact_recv_per_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262888 expected_overflow=0 exact_recv_per_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=163840 expected_recv=261177 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=163840 expected_recv=261834 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262888 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=163840 expected_recv=261770 expected_overflow=0 exact_recv_per_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262288 expected_overflow=0 exact_recv_per_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=163840 expected_recv=261770 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262288 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262427 expected_overflow=0 exact_recv_per_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262016 expected_overflow=0 exact_recv_per_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262132 expected_overflow=0 exact_recv_per_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=163840 expected_recv=262427 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262016 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262114 expected_overflow=0 exact_recv_per_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262132 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262402 expected_overflow=0 exact_recv_per_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262466 expected_overflow=0 exact_recv_per_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262114 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262402 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262466 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262185 expected_overflow=0 exact_recv_per_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=163840 expected_recv=261621 expected_overflow=0 exact_recv_per_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262185 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262216 expected_overflow=0 exact_recv_per_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=163840 expected_recv=261621 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=163840 expected_recv=261680 expected_overflow=0 exact_recv_per_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262908 expected_overflow=0 exact_recv_per_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=163840 expected_recv=262216 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=163840 expected_recv=261680 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262908 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262235 expected_overflow=0 exact_recv_per_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262344 expected_overflow=0 exact_recv_per_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262185 expected_overflow=0 exact_recv_per_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262235 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262344 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262185 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=163840 expected_recv=261506 expected_overflow=0 exact_recv_per_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262481 expected_overflow=0 exact_recv_per_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=163840 expected_recv=261506 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=163840 expected_recv=261813 expected_overflow=0 exact_recv_per_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=163840 expected_recv=262481 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262626 expected_overflow=0 exact_recv_per_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=163840 expected_recv=261619 expected_overflow=0 exact_recv_per_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=163840 expected_recv=261813 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262626 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=163840 expected_recv=261619 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262011 expected_overflow=0 exact_recv_per_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262663 expected_overflow=0 exact_recv_per_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262201 expected_overflow=0 exact_recv_per_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262011 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262663 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262201 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262574 expected_overflow=0 exact_recv_per_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=163840 expected_recv=261694 expected_overflow=0 exact_recv_per_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=163840 expected_recv=261764 expected_overflow=0 exact_recv_per_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=163840 expected_recv=262574 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=163840 expected_recv=261694 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262434 expected_overflow=0 exact_recv_per_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=163840 expected_recv=261764 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262081 expected_overflow=0 exact_recv_per_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262434 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262132 expected_overflow=0 exact_recv_per_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262081 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262438 expected_overflow=0 exact_recv_per_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=163840 expected_recv=261650 expected_overflow=0 exact_recv_per_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262132 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262438 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=163840 expected_recv=261650 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262431 expected_overflow=0 exact_recv_per_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=163840 expected_recv=261560 expected_overflow=0 exact_recv_per_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262426 expected_overflow=0 exact_recv_per_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262431 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=163840 expected_recv=261560 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=163840 expected_recv=262806 expected_overflow=0 exact_recv_per_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=163840 expected_recv=262045 expected_overflow=0 exact_recv_per_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=163840 expected_recv=262426 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=163840 expected_recv=261855 expected_overflow=0 exact_recv_per_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=163840 expected_recv=262806 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=163840 expected_recv=262428 expected_overflow=0 exact_recv_per_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=163840 expected_recv=262045 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=163840 expected_recv=261688 expected_overflow=0 exact_recv_per_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=163840 expected_recv=261855 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=163840 expected_recv=262428 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=163840 expected_recv=261688 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=163840 expected_recv=263130 expected_overflow=0 exact_recv_per_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=163840 expected_recv=262129 expected_overflow=0 exact_recv_per_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=163840 expected_recv=263130 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=163840 expected_recv=262129 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=163840 expected_recv=261071 expected_overflow=0 exact_recv_per_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=163840 expected_recv=262091 expected_overflow=0 exact_recv_per_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=163840 expected_recv=262121 expected_overflow=0 exact_recv_per_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=163840 expected_recv=261937 expected_overflow=0 exact_recv_per_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=163840 expected_recv=261071 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=163840 expected_recv=262091 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=163840 expected_recv=262121 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=163840 expected_recv=261911 expected_overflow=0 exact_recv_per_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=163840 expected_recv=261937 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=163840 expected_recv=261968 expected_overflow=0 exact_recv_per_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=163840 expected_recv=261911 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=163840 expected_recv=261968 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=163840 expected_recv=262715 expected_overflow=0 exact_recv_per_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=163840 expected_recv=262683 expected_overflow=0 exact_recv_per_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=163840 expected_recv=262715 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=163840 expected_recv=261726 expected_overflow=0 exact_recv_per_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=163840 expected_recv=262683 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=163840 expected_recv=261970 expected_overflow=0 exact_recv_per_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262066 expected_overflow=0 exact_recv_per_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=163840 expected_recv=261726 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=163840 expected_recv=261300 expected_overflow=0 exact_recv_per_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=163840 expected_recv=261970 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262066 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=163840 expected_recv=261692 expected_overflow=0 exact_recv_per_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=163840 expected_recv=261300 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262638 expected_overflow=0 exact_recv_per_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=163840 expected_recv=261692 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262638 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262788 expected_overflow=0 exact_recv_per_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262666 expected_overflow=0 exact_recv_per_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262788 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262666 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262032 expected_overflow=0 exact_recv_per_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=163840 expected_recv=261462 expected_overflow=0 exact_recv_per_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262219 expected_overflow=0 exact_recv_per_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=163840 expected_recv=262032 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262630 expected_overflow=0 exact_recv_per_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=163840 expected_recv=261462 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262219 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=163840 expected_recv=261957 expected_overflow=0 exact_recv_per_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262630 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262159 expected_overflow=0 exact_recv_per_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=163840 expected_recv=261957 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262159 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262697 expected_overflow=0 exact_recv_per_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262114 expected_overflow=0 exact_recv_per_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262697 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=163840 expected_recv=262114 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=163840 expected_recv=261914 expected_overflow=0 exact_recv_per_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=163840 expected_recv=262195 expected_overflow=0 exact_recv_per_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=163840 expected_recv=262007 expected_overflow=0 exact_recv_per_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=163840 expected_recv=261914 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261995 expected_overflow=0 exact_recv_per_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=163840 expected_recv=262195 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261975 expected_overflow=0 exact_recv_per_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=163840 expected_recv=262007 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261995 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261967 expected_overflow=0 exact_recv_per_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261975 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=163840 expected_recv=262264 expected_overflow=0 exact_recv_per_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261962 expected_overflow=0 exact_recv_per_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261967 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=163840 expected_recv=262264 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=163840 expected_recv=261962 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=163840 expected_recv=262787 expected_overflow=0 exact_recv_per_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=163840 expected_recv=262787 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[130791, 131244, 131074, 131368, 130871, 131336, 131374, 130518] avg=131072.0000 max=131374 max_rank=6 min=130518 min_rank=7 imbalance_ratio=1.002304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[131077, 131291, 131337, 131036, 130555, 131058, 131325, 130897] avg=131072.0000 max=131337 max_rank=2 min=130555 min_rank=4 imbalance_ratio=1.002022 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[131352, 130900, 131715, 130567, 130717, 130800, 131170, 131355] avg=131072.0000 max=131715 max_rank=2 min=130567 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[131269, 130929, 131257, 130866, 131140, 131508, 130849, 130758] avg=131072.0000 max=131508 max_rank=5 min=130758 min_rank=7 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[130677, 131360, 131156, 131011, 130954, 131338, 130995, 131085] avg=131072.0000 max=131360 max_rank=1 min=130677 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[131600, 130686, 130866, 131094, 131126, 131123, 130961, 131120] avg=131072.0000 max=131600 max_rank=0 min=130686 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[130992, 131127, 130916, 130536, 131186, 130356, 131590, 131873] avg=131072.0000 max=131873 max_rank=7 min=130356 min_rank=5 imbalance_ratio=1.006111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[130890, 131124, 131199, 131448, 131054, 130941, 130892, 131028] avg=131072.0000 max=131448 max_rank=3 min=130890 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[130918, 131132, 131103, 130763, 131033, 131491, 130941, 131195] avg=131072.0000 max=131491 max_rank=5 min=130763 min_rank=3 imbalance_ratio=1.003197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[130492, 131128, 131156, 131025, 130987, 131239, 131158, 131391] avg=131072.0000 max=131391 max_rank=7 min=130492 min_rank=0 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[131055, 130870, 131024, 131229, 131103, 130993, 130944, 131358] avg=131072.0000 max=131358 max_rank=7 min=130870 min_rank=1 imbalance_ratio=1.002182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[130802, 131318, 131095, 131465, 130549, 131086, 131363, 130898] avg=131072.0000 max=131465 max_rank=3 min=130549 min_rank=4 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[131432, 130492, 130858, 131429, 131253, 131222, 130960, 130930] avg=131072.0000 max=131432 max_rank=0 min=130492 min_rank=1 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[131213, 131133, 130657, 131255, 131086, 131006, 131500, 130726] avg=131072.0000 max=131500 max_rank=6 min=130657 min_rank=2 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[130966, 130779, 130924, 130994, 131479, 131286, 131116, 131032] avg=131072.0000 max=131479 max_rank=4 min=130779 min_rank=1 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[131059, 131047, 131000, 130881, 131272, 131092, 131037, 131188] avg=131072.0000 max=131272 max_rank=4 min=130881 min_rank=3 imbalance_ratio=1.001526 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=4194543 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015424 peak_demand=131873 required_capacity=1.006111 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=4196147 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014400 peak_demand=131776 required_capacity=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=4192461 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018112 peak_demand=131600 required_capacity=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=4193493 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016192 peak_demand=131715 required_capacity=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=4195076 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014400 peak_demand=131632 required_capacity=1.004272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=4191716 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018624 peak_demand=131515 required_capacity=1.003380 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=4194908 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016704 peak_demand=131723 required_capacity=1.004967 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=4196088 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014912 peak_demand=131703 required_capacity=1.004814 + [2026-08-30 05:20:25] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 545.6 | throughput per GPU (TFLOP/s/GPU): 611.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.333680E+00 | loss scale: 1.0 | grad norm: 5.472 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 133120 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=163840 expected_recv=261748 expected_overflow=0 exact_recv_per_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=163840 expected_recv=261992 expected_overflow=0 exact_recv_per_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=163840 expected_recv=261876 expected_overflow=0 exact_recv_per_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=163840 expected_recv=262195 expected_overflow=0 exact_recv_per_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=163840 expected_recv=262134 expected_overflow=0 exact_recv_per_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=163840 expected_recv=262539 expected_overflow=0 exact_recv_per_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=163840 expected_recv=261600 expected_overflow=0 exact_recv_per_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=163840 expected_recv=261748 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=163840 expected_recv=261992 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=163840 expected_recv=263068 expected_overflow=0 exact_recv_per_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=163840 expected_recv=261876 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=163840 expected_recv=262195 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=163840 expected_recv=262134 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=163840 expected_recv=262539 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=163840 expected_recv=261600 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=163840 expected_recv=263068 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262303 expected_overflow=0 exact_recv_per_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262199 expected_overflow=0 exact_recv_per_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=163840 expected_recv=261353 expected_overflow=0 exact_recv_per_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262155 expected_overflow=0 exact_recv_per_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=163840 expected_recv=261908 expected_overflow=0 exact_recv_per_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262024 expected_overflow=0 exact_recv_per_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262303 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262199 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262633 expected_overflow=0 exact_recv_per_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=163840 expected_recv=261353 actual_valid_recv=262912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262024 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262155 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=163840 expected_recv=261908 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262577 expected_overflow=0 exact_recv_per_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=262912 slack_slots=64768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262633 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=163840 expected_recv=262577 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261494 expected_overflow=0 exact_recv_per_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=163840 expected_recv=262449 expected_overflow=0 exact_recv_per_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=163840 expected_recv=262411 expected_overflow=0 exact_recv_per_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261494 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=163840 expected_recv=262449 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=163840 expected_recv=262912 expected_overflow=0 exact_recv_per_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=163840 expected_recv=262411 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261482 expected_overflow=0 exact_recv_per_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261225 expected_overflow=0 exact_recv_per_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=163840 expected_recv=262451 expected_overflow=0 exact_recv_per_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=163840 expected_recv=262912 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=163840 expected_recv=262728 expected_overflow=0 exact_recv_per_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261482 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=163840 expected_recv=261225 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=163840 expected_recv=262451 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=163840 expected_recv=262728 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=163840 expected_recv=262982 expected_overflow=0 exact_recv_per_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=163840 expected_recv=261796 expected_overflow=0 exact_recv_per_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=163840 expected_recv=262982 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=163840 expected_recv=261575 expected_overflow=0 exact_recv_per_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=163840 expected_recv=261796 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=163840 expected_recv=261904 expected_overflow=0 exact_recv_per_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=163840 expected_recv=261575 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=163840 expected_recv=262913 expected_overflow=0 exact_recv_per_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=163840 expected_recv=262518 expected_overflow=0 exact_recv_per_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=163840 expected_recv=262048 expected_overflow=0 exact_recv_per_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=163840 expected_recv=261904 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=163840 expected_recv=261416 expected_overflow=0 exact_recv_per_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=163840 expected_recv=262913 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=163840 expected_recv=262518 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=163840 expected_recv=262048 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=163840 expected_recv=261416 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262371 expected_overflow=0 exact_recv_per_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262371 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262573 expected_overflow=0 exact_recv_per_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262011 expected_overflow=0 exact_recv_per_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262573 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=163840 expected_recv=261455 expected_overflow=0 exact_recv_per_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262394 expected_overflow=0 exact_recv_per_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=163840 expected_recv=261935 expected_overflow=0 exact_recv_per_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262011 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=163840 expected_recv=261455 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262267 expected_overflow=0 exact_recv_per_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262146 expected_overflow=0 exact_recv_per_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262394 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=163840 expected_recv=261935 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262267 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=163840 expected_recv=262146 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=163840 expected_recv=262246 expected_overflow=0 exact_recv_per_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=163840 expected_recv=262246 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=163840 expected_recv=261846 expected_overflow=0 exact_recv_per_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=163840 expected_recv=262595 expected_overflow=0 exact_recv_per_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=163840 expected_recv=261552 expected_overflow=0 exact_recv_per_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=163840 expected_recv=261846 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=163840 expected_recv=262584 expected_overflow=0 exact_recv_per_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=163840 expected_recv=262233 expected_overflow=0 exact_recv_per_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=163840 expected_recv=262595 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=163840 expected_recv=261552 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=163840 expected_recv=262174 expected_overflow=0 exact_recv_per_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=163840 expected_recv=261922 expected_overflow=0 exact_recv_per_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=163840 expected_recv=262584 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=163840 expected_recv=262233 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=163840 expected_recv=262174 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=163840 expected_recv=261922 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262044 expected_overflow=0 exact_recv_per_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262044 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=163840 expected_recv=263348 expected_overflow=0 exact_recv_per_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=163840 expected_recv=261286 expected_overflow=0 exact_recv_per_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=163840 expected_recv=263348 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=163840 expected_recv=261286 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=163840 expected_recv=261877 expected_overflow=0 exact_recv_per_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=163840 expected_recv=261751 expected_overflow=0 exact_recv_per_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=163840 expected_recv=261724 expected_overflow=0 exact_recv_per_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262648 expected_overflow=0 exact_recv_per_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=163840 expected_recv=261877 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262474 expected_overflow=0 exact_recv_per_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262153 expected_overflow=0 exact_recv_per_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=163840 expected_recv=261751 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=163840 expected_recv=261724 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262648 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=163840 expected_recv=262474 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262153 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=163840 expected_recv=261711 expected_overflow=0 exact_recv_per_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=163840 expected_recv=261711 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=163840 expected_recv=263072 expected_overflow=0 exact_recv_per_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=163840 expected_recv=263072 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=163840 expected_recv=261800 expected_overflow=0 exact_recv_per_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262310 expected_overflow=0 exact_recv_per_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=163840 expected_recv=261899 expected_overflow=0 exact_recv_per_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262023 expected_overflow=0 exact_recv_per_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=163840 expected_recv=262194 expected_overflow=0 exact_recv_per_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=163840 expected_recv=261800 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262184 expected_overflow=0 exact_recv_per_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262310 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=163840 expected_recv=261899 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262023 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=163840 expected_recv=262194 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=163840 expected_recv=262184 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=163840 expected_recv=262219 expected_overflow=0 exact_recv_per_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=163840 expected_recv=262350 expected_overflow=0 exact_recv_per_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=163840 expected_recv=262219 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=163840 expected_recv=262350 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=163840 expected_recv=261306 expected_overflow=0 exact_recv_per_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=163840 expected_recv=262889 expected_overflow=0 exact_recv_per_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=163840 expected_recv=261674 expected_overflow=0 exact_recv_per_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=163840 expected_recv=262822 expected_overflow=0 exact_recv_per_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=163840 expected_recv=261914 expected_overflow=0 exact_recv_per_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=163840 expected_recv=261306 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=163840 expected_recv=262889 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=163840 expected_recv=262606 expected_overflow=0 exact_recv_per_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=163840 expected_recv=261674 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=163840 expected_recv=262822 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=163840 expected_recv=261914 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=163840 expected_recv=262606 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=163840 expected_recv=261788 expected_overflow=0 exact_recv_per_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=163840 expected_recv=262474 expected_overflow=0 exact_recv_per_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=163840 expected_recv=261788 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=163840 expected_recv=262474 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=163840 expected_recv=261890 expected_overflow=0 exact_recv_per_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=163840 expected_recv=262775 expected_overflow=0 exact_recv_per_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=163840 expected_recv=262045 expected_overflow=0 exact_recv_per_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=163840 expected_recv=261724 expected_overflow=0 exact_recv_per_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=163840 expected_recv=261890 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=163840 expected_recv=261887 expected_overflow=0 exact_recv_per_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=163840 expected_recv=261792 expected_overflow=0 exact_recv_per_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=163840 expected_recv=262045 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=163840 expected_recv=262775 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=163840 expected_recv=261724 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=163840 expected_recv=261887 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=163840 expected_recv=261792 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=163840 expected_recv=262554 expected_overflow=0 exact_recv_per_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=163840 expected_recv=262665 expected_overflow=0 exact_recv_per_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=163840 expected_recv=262554 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=163840 expected_recv=262665 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=163840 expected_recv=262947 expected_overflow=0 exact_recv_per_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=163840 expected_recv=262931 expected_overflow=0 exact_recv_per_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=163840 expected_recv=261402 expected_overflow=0 exact_recv_per_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=163840 expected_recv=261839 expected_overflow=0 exact_recv_per_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=163840 expected_recv=261786 expected_overflow=0 exact_recv_per_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=163840 expected_recv=261914 expected_overflow=0 exact_recv_per_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=163840 expected_recv=262947 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=163840 expected_recv=262931 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=163840 expected_recv=261402 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=163840 expected_recv=261679 expected_overflow=0 exact_recv_per_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=163840 expected_recv=261839 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=163840 expected_recv=261786 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=163840 expected_recv=261914 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=163840 expected_recv=261637 expected_overflow=0 exact_recv_per_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=163840 expected_recv=261679 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=163840 expected_recv=261637 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262035 expected_overflow=0 exact_recv_per_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=163840 expected_recv=261930 expected_overflow=0 exact_recv_per_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=163840 expected_recv=262317 expected_overflow=0 exact_recv_per_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=163840 expected_recv=262659 expected_overflow=0 exact_recv_per_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262035 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=163840 expected_recv=261930 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=163840 expected_recv=262253 expected_overflow=0 exact_recv_per_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=163840 expected_recv=262317 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=163840 expected_recv=261746 expected_overflow=0 exact_recv_per_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=163840 expected_recv=262659 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262124 expected_overflow=0 exact_recv_per_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=163840 expected_recv=262253 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262212 expected_overflow=0 exact_recv_per_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=163840 expected_recv=261746 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262124 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262212 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=163840 expected_recv=261340 expected_overflow=0 exact_recv_per_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=163840 expected_recv=261340 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262450 expected_overflow=0 exact_recv_per_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=163840 expected_recv=261572 expected_overflow=0 exact_recv_per_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262506 expected_overflow=0 exact_recv_per_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262244 expected_overflow=0 exact_recv_per_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262450 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262280 expected_overflow=0 exact_recv_per_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=163840 expected_recv=261572 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262009 expected_overflow=0 exact_recv_per_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262244 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262506 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262508 expected_overflow=0 exact_recv_per_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262280 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=163840 expected_recv=262009 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262508 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262033 expected_overflow=0 exact_recv_per_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262033 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262385 expected_overflow=0 exact_recv_per_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262218 expected_overflow=0 exact_recv_per_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=163840 expected_recv=261495 expected_overflow=0 exact_recv_per_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262273 expected_overflow=0 exact_recv_per_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262295 expected_overflow=0 exact_recv_per_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=163840 expected_recv=261749 expected_overflow=0 exact_recv_per_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262653 expected_overflow=0 exact_recv_per_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262385 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262218 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=163840 expected_recv=261495 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262273 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262295 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=163840 expected_recv=261749 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=163840 expected_recv=262653 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262586 expected_overflow=0 exact_recv_per_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262586 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262007 expected_overflow=0 exact_recv_per_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262467 expected_overflow=0 exact_recv_per_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262007 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262467 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262487 expected_overflow=0 exact_recv_per_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=163840 expected_recv=261496 expected_overflow=0 exact_recv_per_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=163840 expected_recv=261826 expected_overflow=0 exact_recv_per_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262735 expected_overflow=0 exact_recv_per_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262487 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262288 expected_overflow=0 exact_recv_per_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=163840 expected_recv=261496 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=163840 expected_recv=261826 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262735 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=163840 expected_recv=262288 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=163840 expected_recv=261565 expected_overflow=0 exact_recv_per_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262018 expected_overflow=0 exact_recv_per_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=163840 expected_recv=261565 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262018 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262206 expected_overflow=0 exact_recv_per_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262294 expected_overflow=0 exact_recv_per_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262206 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262252 expected_overflow=0 exact_recv_per_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262294 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=163840 expected_recv=262252 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[130906, 131230, 131049, 131026, 131600, 130460, 131482, 130823] avg=131072.0000 max=131600 max_rank=4 min=130460 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[131074, 130857, 131301, 130685, 131389, 131571, 131193, 130506] avg=131072.0000 max=131571 max_rank=5 min=130506 min_rank=7 imbalance_ratio=1.003807 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[131162, 131285, 130909, 131027, 130973, 130615, 131590, 131015] avg=131072.0000 max=131590 max_rank=6 min=130615 min_rank=5 imbalance_ratio=1.003952 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[130682, 131391, 131297, 131284, 130608, 131337, 130825, 131152] avg=131072.0000 max=131391 max_rank=1 min=130608 min_rank=4 imbalance_ratio=1.002434 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[131298, 131147, 130792, 131408, 130781, 131334, 130650, 131166] avg=131072.0000 max=131408 max_rank=3 min=130650 min_rank=6 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[131199, 130616, 131126, 131359, 130706, 131416, 131032, 131122] avg=131072.0000 max=131416 max_rank=5 min=130616 min_rank=1 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[131487, 131559, 131091, 130867, 131200, 130375, 131189, 130808] avg=131072.0000 max=131559 max_rank=1 min=130375 min_rank=5 imbalance_ratio=1.003716 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[130747, 131247, 130804, 130955, 130516, 131346, 131604, 131357] avg=131072.0000 max=131604 max_rank=6 min=130516 min_rank=4 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[131145, 130791, 130341, 131671, 130994, 131282, 131408, 130944] avg=131072.0000 max=131671 max_rank=3 min=130341 min_rank=2 imbalance_ratio=1.004570 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[130961, 130786, 131319, 131298, 130981, 130682, 131365, 131184] avg=131072.0000 max=131365 max_rank=6 min=130682 min_rank=5 imbalance_ratio=1.002235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[131268, 130827, 131551, 130612, 131081, 131001, 131231, 131005] avg=131072.0000 max=131551 max_rank=2 min=130612 min_rank=3 imbalance_ratio=1.003654 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[131181, 131102, 130747, 131193, 131186, 130991, 130612, 131564] avg=131072.0000 max=131564 max_rank=7 min=130612 min_rank=6 imbalance_ratio=1.003754 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[131110, 131257, 131195, 130827, 131240, 131110, 130838, 130999] avg=131072.0000 max=131257 max_rank=1 min=130827 min_rank=3 imbalance_ratio=1.001411 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[130979, 131415, 131107, 131398, 130855, 131332, 130941, 130549] avg=131072.0000 max=131415 max_rank=1 min=130549 min_rank=7 imbalance_ratio=1.002617 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[131145, 130877, 130701, 131298, 131052, 131626, 130912, 130965] avg=131072.0000 max=131626 max_rank=5 min=130701 min_rank=2 imbalance_ratio=1.004227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[130762, 130971, 130877, 130651, 131221, 131171, 131444, 131479] avg=131072.0000 max=131479 max_rank=7 min=130651 min_rank=3 imbalance_ratio=1.003105 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=4194763 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015424 peak_demand=131830 required_capacity=1.005783 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=4196117 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1013120 peak_demand=131604 required_capacity=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=4195384 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015296 peak_demand=131755 required_capacity=1.005211 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=4194209 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015936 peak_demand=131861 required_capacity=1.006020 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=4194509 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015040 peak_demand=131559 required_capacity=1.003716 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=4194377 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014016 peak_demand=131671 required_capacity=1.004570 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=4192643 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018496 peak_demand=131766 required_capacity=1.005295 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=4192430 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017472 peak_demand=131626 required_capacity=1.004227 + [2026-08-30 05:20:25] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 683.5 | throughput per GPU (TFLOP/s/GPU): 487.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.082395E+00 | loss scale: 1.0 | grad norm: 5.378 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 133120 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=163840 expected_recv=262597 expected_overflow=0 exact_recv_per_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=163840 expected_recv=262150 expected_overflow=0 exact_recv_per_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=163840 expected_recv=261819 expected_overflow=0 exact_recv_per_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=163840 expected_recv=262580 expected_overflow=0 exact_recv_per_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=163840 expected_recv=262066 expected_overflow=0 exact_recv_per_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=163840 expected_recv=261345 expected_overflow=0 exact_recv_per_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=163840 expected_recv=262644 expected_overflow=0 exact_recv_per_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=163840 expected_recv=262597 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=163840 expected_recv=262150 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=163840 expected_recv=261951 expected_overflow=0 exact_recv_per_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=163840 expected_recv=261819 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=163840 expected_recv=262580 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=163840 expected_recv=262066 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=163840 expected_recv=261345 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=163840 expected_recv=262644 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=163840 expected_recv=261951 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262382 expected_overflow=0 exact_recv_per_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262054 expected_overflow=0 exact_recv_per_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262297 expected_overflow=0 exact_recv_per_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262382 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262601 expected_overflow=0 exact_recv_per_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262054 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262297 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=163840 expected_recv=261855 expected_overflow=0 exact_recv_per_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262601 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=163840 expected_recv=261381 expected_overflow=0 exact_recv_per_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262322 expected_overflow=0 exact_recv_per_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262260 expected_overflow=0 exact_recv_per_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=163840 expected_recv=261855 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=163840 expected_recv=261381 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262322 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=163840 expected_recv=262260 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=163840 expected_recv=261880 expected_overflow=0 exact_recv_per_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=163840 expected_recv=261880 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262010 expected_overflow=0 exact_recv_per_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262018 expected_overflow=0 exact_recv_per_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262322 expected_overflow=0 exact_recv_per_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262505 expected_overflow=0 exact_recv_per_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262010 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=163840 expected_recv=261891 expected_overflow=0 exact_recv_per_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262018 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262003 expected_overflow=0 exact_recv_per_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262523 expected_overflow=0 exact_recv_per_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262322 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262505 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=163840 expected_recv=261891 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262003 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=163840 expected_recv=262523 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=163840 expected_recv=262171 expected_overflow=0 exact_recv_per_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=163840 expected_recv=262171 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=163840 expected_recv=262517 expected_overflow=0 exact_recv_per_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=163840 expected_recv=261955 expected_overflow=0 exact_recv_per_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=163840 expected_recv=261755 expected_overflow=0 exact_recv_per_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=163840 expected_recv=261945 expected_overflow=0 exact_recv_per_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=163840 expected_recv=262458 expected_overflow=0 exact_recv_per_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=163840 expected_recv=262517 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=163840 expected_recv=261955 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=163840 expected_recv=261755 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=163840 expected_recv=261945 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=163840 expected_recv=262458 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=163840 expected_recv=262560 expected_overflow=0 exact_recv_per_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=163840 expected_recv=261791 expected_overflow=0 exact_recv_per_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=163840 expected_recv=262560 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=163840 expected_recv=261791 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262074 expected_overflow=0 exact_recv_per_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262074 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=163840 expected_recv=261957 expected_overflow=0 exact_recv_per_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=163840 expected_recv=261574 expected_overflow=0 exact_recv_per_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=163840 expected_recv=261957 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=163840 expected_recv=261991 expected_overflow=0 exact_recv_per_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262343 expected_overflow=0 exact_recv_per_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262208 expected_overflow=0 exact_recv_per_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=163840 expected_recv=261574 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=163840 expected_recv=261991 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262343 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262373 expected_overflow=0 exact_recv_per_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262208 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262632 expected_overflow=0 exact_recv_per_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262373 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262073 expected_overflow=0 exact_recv_per_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=163840 expected_recv=262632 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262073 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262777 expected_overflow=0 exact_recv_per_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262777 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=163840 expected_recv=261775 expected_overflow=0 exact_recv_per_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=163840 expected_recv=261718 expected_overflow=0 exact_recv_per_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262019 expected_overflow=0 exact_recv_per_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262348 expected_overflow=0 exact_recv_per_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=163840 expected_recv=261775 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=163840 expected_recv=261718 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262019 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262353 expected_overflow=0 exact_recv_per_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262348 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262353 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262089 expected_overflow=0 exact_recv_per_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=163840 expected_recv=261308 expected_overflow=0 exact_recv_per_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=163840 expected_recv=262089 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262242 expected_overflow=0 exact_recv_per_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=163840 expected_recv=261308 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262242 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262022 expected_overflow=0 exact_recv_per_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262617 expected_overflow=0 exact_recv_per_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262151 expected_overflow=0 exact_recv_per_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262132 expected_overflow=0 exact_recv_per_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262022 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262617 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262151 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262092 expected_overflow=0 exact_recv_per_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262132 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262092 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262588 expected_overflow=0 exact_recv_per_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262086 expected_overflow=0 exact_recv_per_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=163840 expected_recv=261806 expected_overflow=0 exact_recv_per_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=163840 expected_recv=262588 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262086 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=163840 expected_recv=261806 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262431 expected_overflow=0 exact_recv_per_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=163840 expected_recv=261455 expected_overflow=0 exact_recv_per_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=163840 expected_recv=261824 expected_overflow=0 exact_recv_per_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262793 expected_overflow=0 exact_recv_per_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262431 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=163840 expected_recv=261455 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=163840 expected_recv=261824 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262793 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262485 expected_overflow=0 exact_recv_per_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262485 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262272 expected_overflow=0 exact_recv_per_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=163840 expected_recv=262402 expected_overflow=0 exact_recv_per_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261363 expected_overflow=0 exact_recv_per_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=163840 expected_recv=262272 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=163840 expected_recv=262402 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261363 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=163840 expected_recv=262713 expected_overflow=0 exact_recv_per_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261842 expected_overflow=0 exact_recv_per_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261803 expected_overflow=0 exact_recv_per_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=163840 expected_recv=262713 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261803 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=163840 expected_recv=262867 expected_overflow=0 exact_recv_per_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261842 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=163840 expected_recv=262328 expected_overflow=0 exact_recv_per_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=163840 expected_recv=262867 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=163840 expected_recv=262328 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261834 expected_overflow=0 exact_recv_per_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=163840 expected_recv=261938 expected_overflow=0 exact_recv_per_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262357 expected_overflow=0 exact_recv_per_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=163840 expected_recv=261834 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=163840 expected_recv=261938 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262357 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262290 expected_overflow=0 exact_recv_per_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262273 expected_overflow=0 exact_recv_per_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=163840 expected_recv=261769 expected_overflow=0 exact_recv_per_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262290 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262273 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=163840 expected_recv=261769 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=163840 expected_recv=261660 expected_overflow=0 exact_recv_per_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262119 expected_overflow=0 exact_recv_per_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262119 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=163840 expected_recv=261660 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262746 expected_overflow=0 exact_recv_per_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262112 expected_overflow=0 exact_recv_per_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262220 expected_overflow=0 exact_recv_per_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=163840 expected_recv=262746 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262112 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262220 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262199 expected_overflow=0 exact_recv_per_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=163840 expected_recv=261912 expected_overflow=0 exact_recv_per_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262346 expected_overflow=0 exact_recv_per_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262199 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=163840 expected_recv=261912 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262346 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262192 expected_overflow=0 exact_recv_per_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262449 expected_overflow=0 exact_recv_per_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262192 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=163840 expected_recv=262449 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=163840 expected_recv=262127 expected_overflow=0 exact_recv_per_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=163840 expected_recv=261722 expected_overflow=0 exact_recv_per_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=163840 expected_recv=262380 expected_overflow=0 exact_recv_per_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=163840 expected_recv=262127 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=163840 expected_recv=261722 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=163840 expected_recv=262380 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=163840 expected_recv=262596 expected_overflow=0 exact_recv_per_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=163840 expected_recv=261836 expected_overflow=0 exact_recv_per_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=163840 expected_recv=261890 expected_overflow=0 exact_recv_per_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=163840 expected_recv=262596 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=163840 expected_recv=261890 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=163840 expected_recv=261836 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=163840 expected_recv=262018 expected_overflow=0 exact_recv_per_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=163840 expected_recv=261827 expected_overflow=0 exact_recv_per_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262297 expected_overflow=0 exact_recv_per_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=163840 expected_recv=262018 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=163840 expected_recv=262478 expected_overflow=0 exact_recv_per_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=163840 expected_recv=261827 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262297 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262164 expected_overflow=0 exact_recv_per_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=163840 expected_recv=262478 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262164 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=163840 expected_recv=261998 expected_overflow=0 exact_recv_per_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262391 expected_overflow=0 exact_recv_per_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=163840 expected_recv=261651 expected_overflow=0 exact_recv_per_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=163840 expected_recv=261998 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262676 expected_overflow=0 exact_recv_per_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262391 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=163840 expected_recv=261651 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=163840 expected_recv=261702 expected_overflow=0 exact_recv_per_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262287 expected_overflow=0 exact_recv_per_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262676 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=163840 expected_recv=261702 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=163840 expected_recv=262287 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=163840 expected_recv=261688 expected_overflow=0 exact_recv_per_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=163840 expected_recv=261688 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=163840 expected_recv=261982 expected_overflow=0 exact_recv_per_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=163840 expected_recv=262006 expected_overflow=0 exact_recv_per_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=163840 expected_recv=261982 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=163840 expected_recv=262403 expected_overflow=0 exact_recv_per_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=163840 expected_recv=262602 expected_overflow=0 exact_recv_per_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=163840 expected_recv=261748 expected_overflow=0 exact_recv_per_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=163840 expected_recv=262006 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=163840 expected_recv=261677 expected_overflow=0 exact_recv_per_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=163840 expected_recv=262403 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=163840 expected_recv=262602 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=163840 expected_recv=261837 expected_overflow=0 exact_recv_per_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=163840 expected_recv=261748 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=163840 expected_recv=261677 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=163840 expected_recv=261837 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=163840 expected_recv=262872 expected_overflow=0 exact_recv_per_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=163840 expected_recv=262872 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=163840 expected_recv=263001 expected_overflow=0 exact_recv_per_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262735 expected_overflow=0 exact_recv_per_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=163840 expected_recv=261772 expected_overflow=0 exact_recv_per_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=163840 expected_recv=263001 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262735 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=163840 expected_recv=261574 expected_overflow=0 exact_recv_per_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=163840 expected_recv=261772 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262157 expected_overflow=0 exact_recv_per_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=163840 expected_recv=261369 expected_overflow=0 exact_recv_per_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=163840 expected_recv=261738 expected_overflow=0 exact_recv_per_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=163840 expected_recv=261574 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262157 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=163840 expected_recv=261369 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262703 expected_overflow=0 exact_recv_per_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=163840 expected_recv=261738 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=163840 expected_recv=262703 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=163840 expected_recv=261873 expected_overflow=0 exact_recv_per_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=163840 expected_recv=261887 expected_overflow=0 exact_recv_per_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=163840 expected_recv=261873 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=163840 expected_recv=261887 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262438 expected_overflow=0 exact_recv_per_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262438 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262592 expected_overflow=0 exact_recv_per_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=163840 expected_recv=261855 expected_overflow=0 exact_recv_per_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262545 expected_overflow=0 exact_recv_per_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262388 expected_overflow=0 exact_recv_per_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262592 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=163840 expected_recv=261855 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262545 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=163840 expected_recv=262388 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[131437, 131224, 131056, 130839, 131163, 130916, 131200, 130741] avg=131072.0000 max=131437 max_rank=0 min=130741 min_rank=7 imbalance_ratio=1.002785 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[130922, 131389, 130911, 131192, 130854, 131122, 130651, 131535] avg=131072.0000 max=131535 max_rank=7 min=130651 min_rank=6 imbalance_ratio=1.003532 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[130937, 130924, 131366, 131257, 131372, 130529, 130741, 131450] avg=131072.0000 max=131450 max_rank=7 min=130529 min_rank=5 imbalance_ratio=1.002884 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[130870, 130821, 131141, 130670, 131715, 131277, 131195, 130887] avg=131072.0000 max=131715 max_rank=4 min=130670 min_rank=3 imbalance_ratio=1.004906 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[131180, 130322, 131639, 131235, 130849, 131118, 131338, 130895] avg=131072.0000 max=131639 max_rank=2 min=130322 min_rank=1 imbalance_ratio=1.004326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[130961, 130761, 130959, 130943, 131341, 131058, 131171, 131382] avg=131072.0000 max=131382 max_rank=7 min=130761 min_rank=1 imbalance_ratio=1.002365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[131177, 130905, 131612, 130881, 130889, 130773, 131209, 131130] avg=131072.0000 max=131612 max_rank=2 min=130773 min_rank=5 imbalance_ratio=1.004120 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[131204, 130558, 131066, 131124, 131408, 131265, 131371, 130580] avg=131072.0000 max=131408 max_rank=4 min=130558 min_rank=1 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[131120, 131026, 131240, 130785, 131606, 131067, 131421, 130311] avg=131072.0000 max=131606 max_rank=4 min=130311 min_rank=7 imbalance_ratio=1.004074 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[131343, 130914, 131253, 130998, 131060, 131314, 130775, 130919] avg=131072.0000 max=131343 max_rank=0 min=130775 min_rank=6 imbalance_ratio=1.002068 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[131432, 130959, 130887, 130771, 131090, 131286, 131170, 130981] avg=131072.0000 max=131432 max_rank=0 min=130771 min_rank=3 imbalance_ratio=1.002747 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[130831, 131157, 131135, 131627, 130773, 131012, 131045, 130996] avg=131072.0000 max=131627 max_rank=3 min=130773 min_rank=4 imbalance_ratio=1.004234 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[130752, 130979, 131279, 131148, 131121, 131117, 131036, 131144] avg=131072.0000 max=131279 max_rank=2 min=130752 min_rank=0 imbalance_ratio=1.001579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[130559, 131158, 131222, 131557, 131219, 130977, 130850, 131034] avg=131072.0000 max=131557 max_rank=3 min=130559 min_rank=0 imbalance_ratio=1.003700 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[130420, 131496, 131454, 131175, 130905, 131360, 130988, 130778] avg=131072.0000 max=131496 max_rank=1 min=130420 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[130842, 130619, 131501, 131115, 130934, 130989, 131582, 130994] avg=131072.0000 max=131582 max_rank=6 min=130619 min_rank=1 imbalance_ratio=1.003891 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=4192303 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018752 peak_demand=131630 required_capacity=1.004257 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=4192776 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016704 peak_demand=131712 required_capacity=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=4196084 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014400 peak_demand=131715 required_capacity=1.004906 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=4194950 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015936 peak_demand=131681 required_capacity=1.004646 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=4192560 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017728 peak_demand=131679 required_capacity=1.004631 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=4195524 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014272 peak_demand=131582 required_capacity=1.003891 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=4194762 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016192 peak_demand=131639 required_capacity=1.004326 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=4195473 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014272 peak_demand=131748 required_capacity=1.005157 + [2026-08-30 05:20:26] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 751.4 | throughput per GPU (TFLOP/s/GPU): 443.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.928679E+00 | loss scale: 1.0 | grad norm: 5.177 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 133120 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=163840 expected_recv=261515 expected_overflow=0 exact_recv_per_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262064 expected_overflow=0 exact_recv_per_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262088 expected_overflow=0 exact_recv_per_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262314 expected_overflow=0 exact_recv_per_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262446 expected_overflow=0 exact_recv_per_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=163840 expected_recv=261835 expected_overflow=0 exact_recv_per_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262477 expected_overflow=0 exact_recv_per_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=163840 expected_recv=261515 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262064 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262413 expected_overflow=0 exact_recv_per_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262088 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262314 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262446 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=163840 expected_recv=261835 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262477 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=163840 expected_recv=262413 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262177 expected_overflow=0 exact_recv_per_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262391 expected_overflow=0 exact_recv_per_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262201 expected_overflow=0 exact_recv_per_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262078 expected_overflow=0 exact_recv_per_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262177 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=163840 expected_recv=261914 expected_overflow=0 exact_recv_per_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262391 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262201 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=163840 expected_recv=261417 expected_overflow=0 exact_recv_per_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262078 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=163840 expected_recv=261914 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=163840 expected_recv=261417 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262248 expected_overflow=0 exact_recv_per_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262726 expected_overflow=0 exact_recv_per_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262248 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=163840 expected_recv=262726 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=163840 expected_recv=261492 expected_overflow=0 exact_recv_per_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=163840 expected_recv=262300 expected_overflow=0 exact_recv_per_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=163840 expected_recv=262500 expected_overflow=0 exact_recv_per_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=163840 expected_recv=261492 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=163840 expected_recv=262300 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=163840 expected_recv=262500 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=163840 expected_recv=261816 expected_overflow=0 exact_recv_per_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=163840 expected_recv=262417 expected_overflow=0 exact_recv_per_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=163840 expected_recv=262766 expected_overflow=0 exact_recv_per_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=163840 expected_recv=261816 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=163840 expected_recv=262417 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=163840 expected_recv=261945 expected_overflow=0 exact_recv_per_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=163840 expected_recv=262766 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=163840 expected_recv=261916 expected_overflow=0 exact_recv_per_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=163840 expected_recv=261945 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=163840 expected_recv=261916 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=163840 expected_recv=262212 expected_overflow=0 exact_recv_per_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=163840 expected_recv=262621 expected_overflow=0 exact_recv_per_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=163840 expected_recv=262212 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=163840 expected_recv=262621 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=163840 expected_recv=261826 expected_overflow=0 exact_recv_per_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=163840 expected_recv=261592 expected_overflow=0 exact_recv_per_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=163840 expected_recv=261826 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=163840 expected_recv=262828 expected_overflow=0 exact_recv_per_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=163840 expected_recv=261742 expected_overflow=0 exact_recv_per_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=163840 expected_recv=262069 expected_overflow=0 exact_recv_per_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=163840 expected_recv=262262 expected_overflow=0 exact_recv_per_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=163840 expected_recv=261592 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=163840 expected_recv=262828 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=163840 expected_recv=261742 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=163840 expected_recv=262069 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=163840 expected_recv=262262 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=163840 expected_recv=261503 expected_overflow=0 exact_recv_per_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=163840 expected_recv=261503 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262405 expected_overflow=0 exact_recv_per_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262405 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262174 expected_overflow=0 exact_recv_per_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=163840 expected_recv=261480 expected_overflow=0 exact_recv_per_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262174 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262746 expected_overflow=0 exact_recv_per_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262559 expected_overflow=0 exact_recv_per_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=163840 expected_recv=261967 expected_overflow=0 exact_recv_per_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=163840 expected_recv=261480 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262318 expected_overflow=0 exact_recv_per_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262746 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262559 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=163840 expected_recv=261967 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=163840 expected_recv=262318 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=163840 expected_recv=262221 expected_overflow=0 exact_recv_per_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=163840 expected_recv=262551 expected_overflow=0 exact_recv_per_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=163840 expected_recv=262221 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=163840 expected_recv=262551 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=163840 expected_recv=262555 expected_overflow=0 exact_recv_per_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=163840 expected_recv=262555 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=163840 expected_recv=261532 expected_overflow=0 exact_recv_per_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=163840 expected_recv=262260 expected_overflow=0 exact_recv_per_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=163840 expected_recv=261532 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=163840 expected_recv=261886 expected_overflow=0 exact_recv_per_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=163840 expected_recv=262454 expected_overflow=0 exact_recv_per_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=163840 expected_recv=261693 expected_overflow=0 exact_recv_per_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=163840 expected_recv=262260 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=163840 expected_recv=261886 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=163840 expected_recv=262454 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=163840 expected_recv=262301 expected_overflow=0 exact_recv_per_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=163840 expected_recv=261693 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=163840 expected_recv=261707 expected_overflow=0 exact_recv_per_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=163840 expected_recv=262301 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=163840 expected_recv=261707 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=163840 expected_recv=261628 expected_overflow=0 exact_recv_per_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=163840 expected_recv=261628 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=163840 expected_recv=261586 expected_overflow=0 exact_recv_per_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=163840 expected_recv=262814 expected_overflow=0 exact_recv_per_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=163840 expected_recv=263239 expected_overflow=0 exact_recv_per_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=163840 expected_recv=261889 expected_overflow=0 exact_recv_per_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=163840 expected_recv=261586 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=163840 expected_recv=261988 expected_overflow=0 exact_recv_per_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=163840 expected_recv=262814 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=163840 expected_recv=263239 actual_valid_recv=265984 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262272 expected_overflow=0 exact_recv_per_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=163840 expected_recv=261889 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=163840 expected_recv=261707 expected_overflow=0 exact_recv_per_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=163840 expected_recv=261988 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=265984 slack_slots=61696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262272 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=163840 expected_recv=261707 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262328 expected_overflow=0 exact_recv_per_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262328 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=163840 expected_recv=261657 expected_overflow=0 exact_recv_per_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262898 expected_overflow=0 exact_recv_per_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=163840 expected_recv=261980 expected_overflow=0 exact_recv_per_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262738 expected_overflow=0 exact_recv_per_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=163840 expected_recv=261657 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=163840 expected_recv=261572 expected_overflow=0 exact_recv_per_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262898 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=163840 expected_recv=262425 expected_overflow=0 exact_recv_per_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=163840 expected_recv=261980 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=163840 expected_recv=262170 expected_overflow=0 exact_recv_per_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=163840 expected_recv=262738 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=163840 expected_recv=261572 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=163840 expected_recv=262425 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=163840 expected_recv=262170 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=163840 expected_recv=262059 expected_overflow=0 exact_recv_per_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=163840 expected_recv=262059 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=163840 expected_recv=263246 expected_overflow=0 exact_recv_per_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=163840 expected_recv=262301 expected_overflow=0 exact_recv_per_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=163840 expected_recv=261462 expected_overflow=0 exact_recv_per_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=163840 expected_recv=263246 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=163840 expected_recv=261535 expected_overflow=0 exact_recv_per_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=163840 expected_recv=261954 expected_overflow=0 exact_recv_per_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=163840 expected_recv=262301 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=163840 expected_recv=262180 expected_overflow=0 exact_recv_per_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=163840 expected_recv=262363 expected_overflow=0 exact_recv_per_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=163840 expected_recv=261462 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=163840 expected_recv=261535 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=163840 expected_recv=261954 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=163840 expected_recv=262363 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=163840 expected_recv=261964 expected_overflow=0 exact_recv_per_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=163840 expected_recv=262180 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=163840 expected_recv=261964 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=163840 expected_recv=261687 expected_overflow=0 exact_recv_per_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=163840 expected_recv=261715 expected_overflow=0 exact_recv_per_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=163840 expected_recv=261687 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=163840 expected_recv=262835 expected_overflow=0 exact_recv_per_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=163840 expected_recv=261975 expected_overflow=0 exact_recv_per_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=163840 expected_recv=262433 expected_overflow=0 exact_recv_per_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=163840 expected_recv=261715 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262085 expected_overflow=0 exact_recv_per_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262189 expected_overflow=0 exact_recv_per_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262270 expected_overflow=0 exact_recv_per_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=163840 expected_recv=262835 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=163840 expected_recv=261975 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=163840 expected_recv=262433 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262085 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262189 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262270 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262532 expected_overflow=0 exact_recv_per_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262386 expected_overflow=0 exact_recv_per_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=163840 expected_recv=261713 expected_overflow=0 exact_recv_per_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262532 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=163840 expected_recv=261673 expected_overflow=0 exact_recv_per_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262386 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262304 expected_overflow=0 exact_recv_per_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=163840 expected_recv=261971 expected_overflow=0 exact_recv_per_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=163840 expected_recv=261692 expected_overflow=0 exact_recv_per_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=163840 expected_recv=261694 expected_overflow=0 exact_recv_per_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=163840 expected_recv=261713 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=163840 expected_recv=261673 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=163840 expected_recv=261971 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=163840 expected_recv=262304 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=163840 expected_recv=261692 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=163840 expected_recv=261694 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=163840 expected_recv=262906 expected_overflow=0 exact_recv_per_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=163840 expected_recv=261472 expected_overflow=0 exact_recv_per_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=163840 expected_recv=262906 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=163840 expected_recv=261472 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=163840 expected_recv=261993 expected_overflow=0 exact_recv_per_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262013 expected_overflow=0 exact_recv_per_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=163840 expected_recv=262722 expected_overflow=0 exact_recv_per_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262735 expected_overflow=0 exact_recv_per_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=163840 expected_recv=262702 expected_overflow=0 exact_recv_per_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=163840 expected_recv=261900 expected_overflow=0 exact_recv_per_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=163840 expected_recv=261993 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262735 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262013 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=163840 expected_recv=262722 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=163840 expected_recv=262702 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=163840 expected_recv=261900 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262139 expected_overflow=0 exact_recv_per_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=163840 expected_recv=261502 expected_overflow=0 exact_recv_per_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262139 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=163840 expected_recv=261502 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262221 expected_overflow=0 exact_recv_per_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=163840 expected_recv=261800 expected_overflow=0 exact_recv_per_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=163840 expected_recv=262412 expected_overflow=0 exact_recv_per_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262639 expected_overflow=0 exact_recv_per_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262003 expected_overflow=0 exact_recv_per_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=163840 expected_recv=261299 expected_overflow=0 exact_recv_per_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262221 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=163840 expected_recv=261800 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=163840 expected_recv=262412 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262639 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=163840 expected_recv=262003 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=163840 expected_recv=261299 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=163840 expected_recv=262908 expected_overflow=0 exact_recv_per_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=163840 expected_recv=261853 expected_overflow=0 exact_recv_per_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=163840 expected_recv=262908 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=163840 expected_recv=261853 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=163840 expected_recv=262117 expected_overflow=0 exact_recv_per_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=163840 expected_recv=261853 expected_overflow=0 exact_recv_per_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=163840 expected_recv=262258 expected_overflow=0 exact_recv_per_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=163840 expected_recv=262074 expected_overflow=0 exact_recv_per_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=163840 expected_recv=262689 expected_overflow=0 exact_recv_per_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=163840 expected_recv=262117 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=163840 expected_recv=263242 expected_overflow=0 exact_recv_per_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=163840 expected_recv=261853 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=163840 expected_recv=262258 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=163840 expected_recv=262074 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=163840 expected_recv=262689 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=163840 expected_recv=263242 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=163840 expected_recv=262300 expected_overflow=0 exact_recv_per_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=163840 expected_recv=261612 expected_overflow=0 exact_recv_per_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=163840 expected_recv=262300 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=163840 expected_recv=261612 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=163840 expected_recv=262305 expected_overflow=0 exact_recv_per_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=163840 expected_recv=262668 expected_overflow=0 exact_recv_per_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=163840 expected_recv=261362 expected_overflow=0 exact_recv_per_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=163840 expected_recv=261942 expected_overflow=0 exact_recv_per_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=163840 expected_recv=261640 expected_overflow=0 exact_recv_per_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=163840 expected_recv=262305 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=163840 expected_recv=262714 expected_overflow=0 exact_recv_per_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=163840 expected_recv=262668 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=163840 expected_recv=261362 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=163840 expected_recv=261942 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=163840 expected_recv=261640 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=163840 expected_recv=262714 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=163840 expected_recv=262789 expected_overflow=0 exact_recv_per_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=163840 expected_recv=262480 expected_overflow=0 exact_recv_per_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=163840 expected_recv=262789 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=163840 expected_recv=262480 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=163840 expected_recv=261352 expected_overflow=0 exact_recv_per_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=163840 expected_recv=262351 expected_overflow=0 exact_recv_per_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=163840 expected_recv=261436 expected_overflow=0 exact_recv_per_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=163840 expected_recv=261352 actual_valid_recv=263168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=163840 expected_recv=262351 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=163840 expected_recv=261436 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263168 slack_slots=64512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[130702, 130577, 130948, 130684, 131439, 131663, 131200, 131363] avg=131072.0000 max=131663 max_rank=5 min=130577 min_rank=1 imbalance_ratio=1.004509 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[131252, 131181, 130439, 130875, 130906, 131304, 131408, 131211] avg=131072.0000 max=131408 max_rank=6 min=130439 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[131387, 130903, 131387, 130803, 131099, 130875, 130913, 131209] avg=131072.0000 max=131387 max_rank=0 min=130803 min_rank=3 imbalance_ratio=1.002403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[130966, 131340, 130640, 130923, 131062, 131495, 130991, 131159] avg=131072.0000 max=131495 max_rank=5 min=130640 min_rank=2 imbalance_ratio=1.003227 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[131115, 130468, 131276, 130634, 131146, 131648, 131044, 131245] avg=131072.0000 max=131648 max_rank=5 min=130468 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[130927, 131199, 131005, 130837, 130996, 130885, 131317, 131410] avg=131072.0000 max=131410 max_rank=7 min=130837 min_rank=3 imbalance_ratio=1.002579 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[130599, 130834, 131780, 130682, 131277, 131176, 131219, 131009] avg=131072.0000 max=131780 max_rank=2 min=130599 min_rank=0 imbalance_ratio=1.005402 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[131254, 130811, 131179, 130956, 130815, 131203, 131866, 130492] avg=131072.0000 max=131866 max_rank=6 min=130492 min_rank=7 imbalance_ratio=1.006058 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[131052, 131231, 131247, 131363, 131198, 130354, 131163, 130968] avg=131072.0000 max=131363 max_rank=3 min=130354 min_rank=5 imbalance_ratio=1.002220 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[130776, 130793, 131418, 130830, 131238, 131508, 130941, 131072] avg=131072.0000 max=131508 max_rank=5 min=130776 min_rank=0 imbalance_ratio=1.003326 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[131246, 130984, 131277, 131156, 130547, 131147, 131352, 130867] avg=131072.0000 max=131352 max_rank=6 min=130547 min_rank=4 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[130812, 130748, 131139, 131216, 131524, 131543, 130585, 131009] avg=131072.0000 max=131543 max_rank=5 min=130585 min_rank=6 imbalance_ratio=1.003593 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[131179, 131574, 130643, 130888, 131169, 131231, 131110, 130782] avg=131072.0000 max=131574 max_rank=1 min=130643 min_rank=2 imbalance_ratio=1.003830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[130288, 131577, 131175, 131414, 131062, 131146, 130647, 131267] avg=131072.0000 max=131577 max_rank=1 min=130288 min_rank=0 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[131236, 131251, 131210, 131004, 130923, 131116, 130844, 130992] avg=131072.0000 max=131251 max_rank=1 min=130844 min_rank=6 imbalance_ratio=1.001366 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[131456, 130483, 130729, 131576, 130822, 130660, 131279, 131571] avg=131072.0000 max=131576 max_rank=3 min=130483 min_rank=1 imbalance_ratio=1.003845 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=4194503 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016320 peak_demand=131571 required_capacity=1.003807 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=4192946 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017344 peak_demand=131577 required_capacity=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=4194642 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016832 peak_demand=131866 required_capacity=1.006058 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=4194608 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015424 peak_demand=132006 required_capacity=1.007126 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=4193769 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017088 peak_demand=131529 required_capacity=1.003487 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=4194562 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016832 peak_demand=131883 required_capacity=1.006187 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=4193422 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018112 peak_demand=131780 required_capacity=1.005402 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=4195980 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015936 peak_demand=131663 required_capacity=1.004509 + [2026-08-30 05:20:27] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 550.5 | throughput per GPU (TFLOP/s/GPU): 605.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.817042E+00 | loss scale: 1.0 | grad norm: 4.920 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 133120 | min safe capacity: 1.0156 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=163840 expected_recv=261733 expected_overflow=0 exact_recv_per_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=163840 expected_recv=262672 expected_overflow=0 exact_recv_per_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=163840 expected_recv=262776 expected_overflow=0 exact_recv_per_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=163840 expected_recv=261832 expected_overflow=0 exact_recv_per_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=163840 expected_recv=261910 expected_overflow=0 exact_recv_per_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=163840 expected_recv=262375 expected_overflow=0 exact_recv_per_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=163840 expected_recv=262166 expected_overflow=0 exact_recv_per_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=163840 expected_recv=261733 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=163840 expected_recv=262672 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=163840 expected_recv=262776 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=163840 expected_recv=261688 expected_overflow=0 exact_recv_per_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=163840 expected_recv=261832 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=163840 expected_recv=261910 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=163840 expected_recv=262375 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=163840 expected_recv=262166 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=163840 expected_recv=261688 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=163840 expected_recv=262606 expected_overflow=0 exact_recv_per_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=163840 expected_recv=261924 expected_overflow=0 exact_recv_per_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=163840 expected_recv=262395 expected_overflow=0 exact_recv_per_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=163840 expected_recv=261823 expected_overflow=0 exact_recv_per_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=163840 expected_recv=261624 expected_overflow=0 exact_recv_per_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=163840 expected_recv=262663 expected_overflow=0 exact_recv_per_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=163840 expected_recv=262226 expected_overflow=0 exact_recv_per_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=163840 expected_recv=262606 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=163840 expected_recv=261924 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=163840 expected_recv=262395 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=163840 expected_recv=261891 expected_overflow=0 exact_recv_per_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=163840 expected_recv=261823 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=163840 expected_recv=261624 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=163840 expected_recv=262663 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=163840 expected_recv=262226 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=163840 expected_recv=261891 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=163840 expected_recv=261819 expected_overflow=0 exact_recv_per_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=163840 expected_recv=261754 expected_overflow=0 exact_recv_per_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262110 expected_overflow=0 exact_recv_per_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262111 expected_overflow=0 exact_recv_per_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262406 expected_overflow=0 exact_recv_per_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=163840 expected_recv=261819 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262112 expected_overflow=0 exact_recv_per_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=163840 expected_recv=261754 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262110 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262111 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262406 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262013 expected_overflow=0 exact_recv_per_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262827 expected_overflow=0 exact_recv_per_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262112 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262013 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=163840 expected_recv=262827 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=163840 expected_recv=261499 expected_overflow=0 exact_recv_per_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=163840 expected_recv=261877 expected_overflow=0 exact_recv_per_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=163840 expected_recv=261499 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=163840 expected_recv=261877 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=163840 expected_recv=262581 expected_overflow=0 exact_recv_per_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=163840 expected_recv=262263 expected_overflow=0 exact_recv_per_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=163840 expected_recv=262811 expected_overflow=0 exact_recv_per_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=163840 expected_recv=262075 expected_overflow=0 exact_recv_per_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=163840 expected_recv=262581 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=163840 expected_recv=262081 expected_overflow=0 exact_recv_per_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=163840 expected_recv=262263 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=163840 expected_recv=262811 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=163840 expected_recv=261965 expected_overflow=0 exact_recv_per_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=163840 expected_recv=262075 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=163840 expected_recv=262081 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=163840 expected_recv=261965 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=163840 expected_recv=261024 expected_overflow=0 exact_recv_per_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262023 expected_overflow=0 exact_recv_per_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=163840 expected_recv=261024 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262023 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=163840 expected_recv=261919 expected_overflow=0 exact_recv_per_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262879 expected_overflow=0 exact_recv_per_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262240 expected_overflow=0 exact_recv_per_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262524 expected_overflow=0 exact_recv_per_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=163840 expected_recv=261919 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262298 expected_overflow=0 exact_recv_per_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262879 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262524 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262240 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262245 expected_overflow=0 exact_recv_per_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262298 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=163840 expected_recv=262245 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=163840 expected_recv=261803 expected_overflow=0 exact_recv_per_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=163840 expected_recv=261835 expected_overflow=0 exact_recv_per_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=163840 expected_recv=261803 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=163840 expected_recv=261835 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=163840 expected_recv=262905 expected_overflow=0 exact_recv_per_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=163840 expected_recv=262565 expected_overflow=0 exact_recv_per_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=163840 expected_recv=261920 expected_overflow=0 exact_recv_per_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=163840 expected_recv=262199 expected_overflow=0 exact_recv_per_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=163840 expected_recv=262905 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=163840 expected_recv=262086 expected_overflow=0 exact_recv_per_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=163840 expected_recv=262565 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=163840 expected_recv=261920 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=163840 expected_recv=262199 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=163840 expected_recv=262086 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=163840 expected_recv=261839 expected_overflow=0 exact_recv_per_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=163840 expected_recv=262617 expected_overflow=0 exact_recv_per_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=163840 expected_recv=261839 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=163840 expected_recv=261554 expected_overflow=0 exact_recv_per_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=163840 expected_recv=262617 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=163840 expected_recv=261554 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=163840 expected_recv=263026 expected_overflow=0 exact_recv_per_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=163840 expected_recv=262313 expected_overflow=0 exact_recv_per_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=163840 expected_recv=263026 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=163840 expected_recv=262692 expected_overflow=0 exact_recv_per_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=163840 expected_recv=262313 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=163840 expected_recv=261056 expected_overflow=0 exact_recv_per_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=163840 expected_recv=262202 expected_overflow=0 exact_recv_per_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=163840 expected_recv=262692 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=163840 expected_recv=261056 actual_valid_recv=262784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=163840 expected_recv=261692 expected_overflow=0 exact_recv_per_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=163840 expected_recv=262202 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=262784 slack_slots=64896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=163840 expected_recv=261874 expected_overflow=0 exact_recv_per_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=163840 expected_recv=262354 expected_overflow=0 exact_recv_per_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=163840 expected_recv=261692 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=163840 expected_recv=261874 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=163840 expected_recv=262354 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=163840 expected_recv=261649 expected_overflow=0 exact_recv_per_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=163840 expected_recv=261589 expected_overflow=0 exact_recv_per_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=163840 expected_recv=261649 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=163840 expected_recv=261802 expected_overflow=0 exact_recv_per_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=163840 expected_recv=261589 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=163840 expected_recv=262327 expected_overflow=0 exact_recv_per_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=163840 expected_recv=263102 expected_overflow=0 exact_recv_per_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=163840 expected_recv=261802 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=163840 expected_recv=262327 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=163840 expected_recv=263102 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262241 expected_overflow=0 exact_recv_per_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=163840 expected_recv=262455 expected_overflow=0 exact_recv_per_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262267 expected_overflow=0 exact_recv_per_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262241 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=163840 expected_recv=262455 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262267 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=163840 expected_recv=261446 expected_overflow=0 exact_recv_per_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262514 expected_overflow=0 exact_recv_per_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=163840 expected_recv=261998 expected_overflow=0 exact_recv_per_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262451 expected_overflow=0 exact_recv_per_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262011 expected_overflow=0 exact_recv_per_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=163840 expected_recv=261446 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262514 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=163840 expected_recv=261998 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262451 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262011 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=163840 expected_recv=261831 expected_overflow=0 exact_recv_per_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262224 expected_overflow=0 exact_recv_per_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=163840 expected_recv=262530 expected_overflow=0 exact_recv_per_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=163840 expected_recv=261831 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=163840 expected_recv=262224 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=163840 expected_recv=262530 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=163840 expected_recv=262418 expected_overflow=0 exact_recv_per_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=163840 expected_recv=261682 expected_overflow=0 exact_recv_per_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=163840 expected_recv=261557 expected_overflow=0 exact_recv_per_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=163840 expected_recv=261898 expected_overflow=0 exact_recv_per_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=163840 expected_recv=262418 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=163840 expected_recv=262788 expected_overflow=0 exact_recv_per_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=163840 expected_recv=261682 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=163840 expected_recv=261898 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=163840 expected_recv=261557 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=163840 expected_recv=262788 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=163840 expected_recv=261936 expected_overflow=0 exact_recv_per_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=163840 expected_recv=262448 expected_overflow=0 exact_recv_per_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=163840 expected_recv=261936 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=163840 expected_recv=262500 expected_overflow=0 exact_recv_per_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=163840 expected_recv=262448 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=163840 expected_recv=262500 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=163840 expected_recv=262163 expected_overflow=0 exact_recv_per_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=163840 expected_recv=262687 expected_overflow=0 exact_recv_per_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=163840 expected_recv=261639 expected_overflow=0 exact_recv_per_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=163840 expected_recv=262163 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=163840 expected_recv=262289 expected_overflow=0 exact_recv_per_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=163840 expected_recv=262402 expected_overflow=0 exact_recv_per_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=163840 expected_recv=262687 actual_valid_recv=264960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=163840 expected_recv=261639 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=163840 expected_recv=262289 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262092 expected_overflow=0 exact_recv_per_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264960 slack_slots=62720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=163840 expected_recv=262402 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=163840 expected_recv=261536 expected_overflow=0 exact_recv_per_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262092 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=163840 expected_recv=261778 expected_overflow=0 exact_recv_per_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=163840 expected_recv=261536 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=163840 expected_recv=261778 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262639 expected_overflow=0 exact_recv_per_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262170 expected_overflow=0 exact_recv_per_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262607 expected_overflow=0 exact_recv_per_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262639 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262079 expected_overflow=0 exact_recv_per_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262170 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262113 expected_overflow=0 exact_recv_per_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262607 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261972 expected_overflow=0 exact_recv_per_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262079 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=163840 expected_recv=261674 expected_overflow=0 exact_recv_per_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=163840 expected_recv=262113 actual_valid_recv=264576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261972 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=163840 expected_recv=262241 expected_overflow=0 exact_recv_per_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=163840 expected_recv=261674 actual_valid_recv=263296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=264576 slack_slots=63104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=163840 expected_recv=262241 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=327680 valid_num_tokens=263296 slack_slots=64384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261396 expected_overflow=0 exact_recv_per_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=163840 expected_recv=262081 expected_overflow=0 exact_recv_per_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261930 expected_overflow=0 exact_recv_per_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261396 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=163840 expected_recv=262081 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=163840 expected_recv=263223 expected_overflow=0 exact_recv_per_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261930 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=163840 expected_recv=261917 expected_overflow=0 exact_recv_per_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261898 expected_overflow=0 exact_recv_per_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=163840 expected_recv=263223 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=163840 expected_recv=262411 expected_overflow=0 exact_recv_per_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=163840 expected_recv=261917 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=163840 expected_recv=261898 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=163840 expected_recv=262412 expected_overflow=0 exact_recv_per_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=163840 expected_recv=262411 actual_valid_recv=264320 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=163840 expected_recv=262412 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=327680 valid_num_tokens=264320 slack_slots=63360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=163840 expected_recv=261781 expected_overflow=0 exact_recv_per_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=163840 expected_recv=261278 expected_overflow=0 exact_recv_per_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=163840 expected_recv=261781 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=163840 expected_recv=261278 actual_valid_recv=263040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=163840 expected_recv=263120 expected_overflow=0 exact_recv_per_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=163840 expected_recv=262109 expected_overflow=0 exact_recv_per_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261838 expected_overflow=0 exact_recv_per_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263040 slack_slots=64640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=163840 expected_recv=262109 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=163840 expected_recv=263120 actual_valid_recv=265216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=163840 expected_recv=262363 expected_overflow=0 exact_recv_per_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=163840 expected_recv=262172 expected_overflow=0 exact_recv_per_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261838 actual_valid_recv=263424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261894 expected_overflow=0 exact_recv_per_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=265216 slack_slots=62464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263424 slack_slots=64256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=163840 expected_recv=262363 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=163840 expected_recv=262172 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261894 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261845 expected_overflow=0 exact_recv_per_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261845 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=163840 expected_recv=262335 expected_overflow=0 exact_recv_per_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261920 expected_overflow=0 exact_recv_per_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=163840 expected_recv=262053 expected_overflow=0 exact_recv_per_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261764 expected_overflow=0 exact_recv_per_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=163840 expected_recv=262335 actual_valid_recv=264704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=163840 expected_recv=261920 actual_valid_recv=263552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=163840 expected_recv=262053 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264704 slack_slots=62976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=163840 expected_recv=262623 expected_overflow=0 exact_recv_per_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261764 actual_valid_recv=264064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=163840 expected_recv=262644 expected_overflow=0 exact_recv_per_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261919 expected_overflow=0 exact_recv_per_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=263552 slack_slots=64128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264064 slack_slots=63616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=163840 expected_recv=262623 actual_valid_recv=264832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=163840 expected_recv=262644 actual_valid_recv=264448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261919 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=163840 expected_recv=262236 expected_overflow=0 exact_recv_per_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264832 slack_slots=62848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=327680 valid_num_tokens=264448 slack_slots=63232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=163840 expected_recv=262236 actual_valid_recv=264192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=264192 slack_slots=63488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261842 expected_overflow=0 exact_recv_per_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=163840 expected_recv=262827 expected_overflow=0 exact_recv_per_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=163840 expected_recv=263230 expected_overflow=0 exact_recv_per_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261842 actual_valid_recv=263936 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=163840 expected_recv=262827 actual_valid_recv=265088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=163840 expected_recv=263230 actual_valid_recv=265344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263936 slack_slots=63744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261481 expected_overflow=0 exact_recv_per_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=265088 slack_slots=62592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261853 expected_overflow=0 exact_recv_per_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=265344 slack_slots=62336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261481 actual_valid_recv=263680 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=163840 expected_recv=261853 actual_valid_recv=263808 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263680 slack_slots=64000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=327680 valid_num_tokens=263808 slack_slots=63872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[130988, 130489, 131028, 131759, 131053, 130916, 130868, 131475] avg=131072.0000 max=131759 max_rank=3 min=130489 min_rank=1 imbalance_ratio=1.005241 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[130770, 130656, 131279, 131238, 130622, 131570, 131317, 131124] avg=131072.0000 max=131570 max_rank=5 min=130622 min_rank=4 imbalance_ratio=1.003799 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[131375, 131018, 130900, 130982, 130879, 131360, 131303, 130759] avg=131072.0000 max=131375 max_rank=0 min=130759 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[131108, 131217, 131091, 131142, 131068, 131373, 130791, 130786] avg=131072.0000 max=131373 max_rank=5 min=130786 min_rank=7 imbalance_ratio=1.002296 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[131262, 131253, 131094, 131375, 131220, 130789, 131042, 130541] avg=131072.0000 max=131375 max_rank=3 min=130541 min_rank=7 imbalance_ratio=1.002312 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[130770, 130715, 130866, 131686, 130781, 130964, 131747, 131047] avg=131072.0000 max=131747 max_rank=6 min=130715 min_rank=1 imbalance_ratio=1.005150 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[131320, 130120, 130971, 131392, 131041, 130840, 131428, 131464] avg=131072.0000 max=131464 max_rank=7 min=130120 min_rank=1 imbalance_ratio=1.002991 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[131140, 130864, 131283, 130475, 131678, 131394, 130987, 130755] avg=131072.0000 max=131678 max_rank=4 min=130475 min_rank=3 imbalance_ratio=1.004623 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[131154, 130864, 131446, 131237, 131061, 131322, 130720, 130772] avg=131072.0000 max=131446 max_rank=2 min=130720 min_rank=6 imbalance_ratio=1.002853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[131451, 130596, 131203, 130736, 131012, 131407, 131465, 130706] avg=131072.0000 max=131465 max_rank=6 min=130596 min_rank=1 imbalance_ratio=1.002998 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[131082, 131288, 131410, 131601, 130751, 130275, 130856, 131313] avg=131072.0000 max=131601 max_rank=3 min=130275 min_rank=5 imbalance_ratio=1.004036 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[130957, 130964, 130724, 131114, 131534, 130721, 131369, 131193] avg=131072.0000 max=131534 max_rank=4 min=130721 min_rank=5 imbalance_ratio=1.003525 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[131250, 130588, 131134, 131658, 131447, 130820, 130608, 131071] avg=131072.0000 max=131658 max_rank=3 min=130588 min_rank=1 imbalance_ratio=1.004471 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[130818, 130567, 131007, 131400, 131319, 130995, 131461, 131009] avg=131072.0000 max=131461 max_rank=6 min=130567 min_rank=1 imbalance_ratio=1.002968 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[131292, 131532, 130948, 130536, 130572, 131577, 130935, 131184] avg=131072.0000 max=131577 max_rank=5 min=130536 min_rank=3 imbalance_ratio=1.003853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[130726, 131167, 130750, 131054, 131695, 130859, 130981, 131344] avg=131072.0000 max=131695 max_rank=4 min=130726 min_rank=0 imbalance_ratio=1.004753 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=4190927 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1019776 peak_demand=131475 required_capacity=1.003075 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=4195239 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014656 peak_demand=131790 required_capacity=1.005478 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=4196289 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1013248 peak_demand=131801 required_capacity=1.005562 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=4191947 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1018240 peak_demand=131594 required_capacity=1.003983 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=4193277 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1017344 peak_demand=131446 required_capacity=1.002853 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=4194863 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1015424 peak_demand=131545 required_capacity=1.003609 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=4194586 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1016576 peak_demand=131591 required_capacity=1.003960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=4197304 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=1014528 peak_demand=131773 required_capacity=1.005348 + [2026-08-30 05:20:27] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 548.1 | throughput per GPU (TFLOP/s/GPU): 608.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.701100E+00 | loss scale: 1.0 | grad norm: 4.790 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 133120 | min safe capacity: 1.0156 | +[after training is done] datetime: 2026-08-30 05:20:27 +[rank0]:[W830 05:20:28.049828994 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 05:20:30.495864006 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 05:20:30.500857138 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 05:20:30.502355360 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 05:20:30.502560577 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 05:20:30.503304142 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 05:20:30.504511326 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 05:20:30.711277935 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:20:34.425000 512593 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:20:34.425000 512593 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 512685 closing signal SIGTERM +W0830 05:20:34.426000 512593 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 512686 closing signal SIGTERM +W0830 05:20:34.426000 512593 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 512687 closing signal SIGTERM +W0830 05:20:34.426000 512593 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 512688 closing signal SIGTERM +W0830 05:20:34.427000 512593 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 512689 closing signal SIGTERM +W0830 05:20:34.427000 512593 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 512690 closing signal SIGTERM +W0830 05:20:34.427000 512593 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 512691 closing signal SIGTERM +W0830 05:20:34.428000 512593 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 512692 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 512593 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv index 521ff829b84d8b20fea1f26ede308016fee6a30d..0016dc9f333a73963a39a9d0c40233b65a45dc95 100644 --- a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv @@ -3,3 +3,10 @@ G1 ultraep measure 1.25 OK 108731 0 0.6209174791971842 1.724609 /workspace/tiera G1 ultraep validate 1.8125 OK 113767 0 0.032329559326171875 2.309082 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 G2 ultraep measure 1.25 OK 110343 1 0.0 1.010101 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1 G2 ultraep validate 1.0625 OK 107987 1 0.0 1.010101 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 +G2 ultraep forced 1.25 OK 110343 1 0.0 1.01001 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 +G3 ultraep measure 1.25 OK 114319 1 0.0 1.010056 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1 +G3 ultraep validate 1.0625 OK 111567 1 0.0 1.010071 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 +G3 ultraep forced 1.25 OK 114267 1 0.0 1.01001 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 +G4 ultraep measure 1.25 OK 120487 1 0.0 1.042938 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1 +G4 ultraep validate 1.095703125 OK 118667 1 0.0 1.042938 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1 +G4 ultraep forced 1.25 OK 120487 1 0.0 1.007126 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv index 6f65ef9c3cf1309f19e8cc4d1196b93d38a9d31f..cc3a6c4c3b0ba40c25a7356d2be4c05329fbe2a3 100644 --- a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv @@ -1,2 +1,5 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir G1 ultraep ROUNDED_CAPACITY_VALIDATION_DROPPED +G2 ultraep SELECTED 1.0625 1.010101 0.0 107987 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 +G3 ultraep SELECTED 1.0625 1.010071 0.0 111567 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 +G4 ultraep SELECTED 1.095703125 1.042938 0.0 118667 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.json b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ff478b204692dded5b5b43fc7c20b17be9c2b3d6 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.json @@ -0,0 +1,849 @@ +[ + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G1", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 25165824.0, + "drop_rate_pct_max": 2.891776, + "drop_rate_pct_mean": 0.607152, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.6209174791971842, + "dropped_tokens_sum_all_records": 156259.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "1", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "8", + "num_layers": "16", + "observed_dropped_tokens": 156259.0, + "peak_demand_max": 14128.0, + "peak_demand_mean": 10918.34375, + "peak_demand_min": 8275.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 3888.0, + "peak_rank_overflow_mean": 1076.4583333333333, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.6189049804826644, + "post_physical_rank_cv_mean": 0.08780981215252336, + "post_physical_rank_cv_min": 0.015802566559489504, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.724609375, + "post_physical_rank_imax_mean": 1.063955307006836, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 1.7320508075688772, + "pre_home_rank_cv_mean": 1.4485647787782712, + "pre_home_rank_cv_min": 0.3594758110042677, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 4.0, + "pre_home_rank_imax_mean": 3.6084569295247397, + "pre_home_rank_imax_min": 1.6087646484375, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 1.724609, + "required_capacity_mean": 1.33280565625, + "required_capacity_min": 1.010132, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "2", + "valid_max_rank_reduction_max": 0.7474516266861991, + "valid_max_rank_reduction_mean": 0.6845405986286021, + "valid_max_rank_reduction_min": 0.27725927612110174, + "valid_max_rank_reduction_n": 192, + "zero_drop": 0 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.8125", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G1", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 25165824.0, + "drop_rate_pct_max": 1.545951, + "drop_rate_pct_mean": 0.03138885416666667, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.032329559326171875, + "dropped_tokens_sum_all_records": 8136.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "1", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "8", + "num_layers": "16", + "observed_dropped_tokens": 8136.0, + "peak_demand_max": 18916.0, + "peak_demand_mean": 11069.770833333334, + "peak_demand_min": 8275.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 4068.0, + "peak_rank_overflow_mean": 84.75, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 1.0236043430060333, + "post_physical_rank_cv_mean": 0.09497773141059589, + "post_physical_rank_cv_min": 0.015224448662595366, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 2.30908203125, + "post_physical_rank_imax_mean": 1.0747025807698567, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 1.7320508075688772, + "pre_home_rank_cv_mean": 1.444809403446171, + "pre_home_rank_cv_min": 0.3594758110042677, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 4.0, + "pre_home_rank_imax_mean": 3.6151854197184243, + "pre_home_rank_imax_min": 1.6087646484375, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 2.309082, + "required_capacity_mean": 1.3512904166666668, + "required_capacity_min": 1.010132, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "2", + "valid_max_rank_reduction_max": 0.7474516266861991, + "valid_max_rank_reduction_mean": 0.6828229027074976, + "valid_max_rank_reduction_min": 0.27725927612110174, + "valid_max_rank_reduction_n": 192, + "zero_drop": 0 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.0625", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G2", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 100663296.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "4", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "32", + "num_layers": "16", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33099.0, + "peak_demand_mean": 33096.927083333336, + "peak_demand_min": 33095.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.026494891363095952, + "post_physical_rank_cv_mean": 0.013436746135527637, + "post_physical_rank_cv_min": 0.004813851711632, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.010101318359375, + "post_physical_rank_imax_mean": 1.0100231170654297, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 1.3185407730096428, + "pre_home_rank_cv_mean": 0.7088423169085597, + "pre_home_rank_cv_min": 0.07964958686803787, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 3.99853515625, + "pre_home_rank_imax_mean": 2.2032313346862793, + "pre_home_rank_imax_min": 1.17706298828125, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 1.010101, + "required_capacity_mean": 1.0100380104166666, + "required_capacity_min": 1.009979, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7474050555623397, + "valid_max_rank_reduction_mean": 0.5179521802733736, + "valid_max_rank_reduction_min": 0.14192377495462796, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G2", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 100663296.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "4", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "32", + "num_layers": "16", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33099.0, + "peak_demand_mean": 33096.875, + "peak_demand_min": 33096.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.026494891363095952, + "post_physical_rank_cv_mean": 0.013604201092033112, + "post_physical_rank_cv_min": 0.004841919368473151, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.01007080078125, + "post_physical_rank_imax_mean": 1.0100212097167969, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 1.3181329746469168, + "pre_home_rank_cv_mean": 0.7143701949255296, + "pre_home_rank_cv_min": 0.22890444224298007, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 3.99853515625, + "pre_home_rank_imax_mean": 2.2135567665100098, + "pre_home_rank_imax_min": 1.33551025390625, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 1.010101, + "required_capacity_mean": 1.0100364479166666, + "required_capacity_min": 1.01001, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7474050555623397, + "valid_max_rank_reduction_mean": 0.5202877868268106, + "valid_max_rank_reduction_min": 0.24370458388556282, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G2", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 100663296.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "4", + "force_balance": "1", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "32", + "num_layers": "16", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33096.0, + "peak_demand_mean": 33052.15625, + "peak_demand_min": 32901.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.00717872106815807, + "post_physical_rank_cv_mean": 0.004277032344446821, + "post_physical_rank_cv_min": 0.0016880800477701613, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.010009765625, + "post_physical_rank_imax_mean": 1.006682554880778, + "post_physical_rank_imax_min": 1.002349853515625, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 0.007852555942790041, + "pre_home_rank_cv_mean": 0.00436334506476056, + "pre_home_rank_cv_min": 0.0016880800477701613, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 1.01544189453125, + "pre_home_rank_imax_mean": 1.0068944295247395, + "pre_home_rank_imax_min": 1.002349853515625, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 1.01001, + "required_capacity_mean": 1.0086718541666666, + "required_capacity_min": 1.004059, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.005349522149425978, + "valid_max_rank_reduction_mean": 0.0002090865394183532, + "valid_max_rank_reduction_min": 0.0, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.0625", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G3", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 201326592.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "64", + "num_layers": "16", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 66196.0, + "peak_demand_mean": 66193.28125, + "peak_demand_min": 66192.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.026489120474252784, + "post_physical_rank_cv_mean": 0.014233627693615905, + "post_physical_rank_cv_min": 0.004993448920418017, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.01007080078125, + "post_physical_rank_imax_mean": 1.0100196202596028, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 0.8287584673770169, + "pre_home_rank_cv_mean": 0.47726214620952834, + "pre_home_rank_cv_min": 0.11019481148220499, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 2.6106719970703125, + "pre_home_rank_imax_mean": 1.7842089335123699, + "pre_home_rank_imax_min": 1.1877593994140625, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 1.010071, + "required_capacity_mean": 1.0100292499999999, + "required_capacity_min": 1.01001, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "16", + "valid_max_rank_reduction_max": 0.6131226876610966, + "valid_max_rank_reduction_mean": 0.41320151666180793, + "valid_max_rank_reduction_min": 0.14965121208617566, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G3", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 201326592.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "64", + "num_layers": "16", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 66195.0, + "peak_demand_mean": 66193.34375, + "peak_demand_min": 66192.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.026483352723689336, + "post_physical_rank_cv_mean": 0.014464507564652654, + "post_physical_rank_cv_min": 0.00571681991985643, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.0100555419921875, + "post_physical_rank_imax_mean": 1.0100204149882, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 0.8316092132265209, + "pre_home_rank_cv_mean": 0.4753321564816673, + "pre_home_rank_cv_min": 0.11019481148220499, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 2.8477020263671875, + "pre_home_rank_imax_mean": 1.7751669883728027, + "pre_home_rank_imax_min": 1.1877593994140625, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 1.010056, + "required_capacity_mean": 1.01003021875, + "required_capacity_min": 1.01001, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "16", + "valid_max_rank_reduction_max": 0.6453085566397145, + "valid_max_rank_reduction_mean": 0.41069380212188733, + "valid_max_rank_reduction_min": 0.14965121208617566, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G3", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 201326592.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "8", + "force_balance": "1", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "64", + "num_layers": "16", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 66192.0, + "peak_demand_mean": 65961.36458333333, + "peak_demand_min": 65746.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.005936610286960278, + "post_physical_rank_cv_mean": 0.0030208295858681123, + "post_physical_rank_cv_min": 0.0014995787955522665, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.010009765625, + "post_physical_rank_imax_mean": 1.004675070444743, + "post_physical_rank_imax_min": 1.001800537109375, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 0.005936610286960278, + "pre_home_rank_cv_mean": 0.0030232844076121573, + "pre_home_rank_cv_min": 0.0014995787955522665, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 1.0113067626953125, + "pre_home_rank_imax_mean": 1.0046818256378174, + "pre_home_rank_imax_min": 1.001800537109375, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 1.01001, + "required_capacity_mean": 1.0064905625, + "required_capacity_min": 1.003204, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "16", + "valid_max_rank_reduction_max": 0.0012824961902319055, + "valid_max_rank_reduction_mean": 6.679667657457841e-06, + "valid_max_rank_reduction_min": 0.0, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.095703125", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G4", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 402653184.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "16", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "128", + "num_layers": "16", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 136700.0, + "peak_demand_mean": 132779.11458333334, + "peak_demand_min": 132383.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.026463165595102558, + "post_physical_rank_cv_mean": 0.014430625474517489, + "post_physical_rank_cv_min": 0.005656712434847103, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.0100250244140625, + "post_physical_rank_imax_mean": 1.0100077788035076, + "post_physical_rank_imax_min": 1.0100021362304688, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 0.607811574382448, + "pre_home_rank_cv_mean": 0.34190630383555115, + "pre_home_rank_cv_min": 0.08731759963810155, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 2.2518386840820312, + "pre_home_rank_imax_mean": 1.561224063237508, + "pre_home_rank_imax_min": 1.121063232421875, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 1.042938, + "required_capacity_mean": 1.0130241979166665, + "required_capacity_min": 1.010002, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "32", + "valid_max_rank_reduction_max": 0.5514766917497027, + "valid_max_rank_reduction_mean": 0.3391114043117762, + "valid_max_rank_reduction_min": 0.099060841159657, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G4", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 402653184.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "16", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "128", + "num_layers": "16", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 136700.0, + "peak_demand_mean": 132713.30208333334, + "peak_demand_min": 132383.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.02646316532015595, + "post_physical_rank_cv_mean": 0.01471717424328191, + "post_physical_rank_cv_min": 0.0057054839456758865, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.0100326538085938, + "post_physical_rank_imax_mean": 1.0100078185399373, + "post_physical_rank_imax_min": 1.0100021362304688, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 0.6043235604226133, + "pre_home_rank_cv_mean": 0.3401136548111448, + "pre_home_rank_cv_min": 0.08731759963810155, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 2.2515869140625, + "pre_home_rank_imax_mean": 1.5504355827967327, + "pre_home_rank_imax_min": 1.121063232421875, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 1.042938, + "required_capacity_mean": 1.0125220937500001, + "required_capacity_min": 1.010002, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "32", + "valid_max_rank_reduction_max": 0.5514197614529683, + "valid_max_rank_reduction_mean": 0.3357737985870543, + "valid_max_rank_reduction_min": 0.099060841159657, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.25", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "G4", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 402653184.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "16", + "force_balance": "1", + "global_batch_size": "16", + "global_snapshots": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "128", + "num_layers": "16", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 132006.0, + "peak_demand_mean": 131665.52083333334, + "peak_demand_min": 131398.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.00357507965332711, + "post_physical_rank_cv_mean": 0.0021567674165372404, + "post_physical_rank_cv_min": 0.0006899225660317828, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.0064849853515625, + "post_physical_rank_imax_mean": 1.0033422311147053, + "post_physical_rank_imax_min": 1.0009918212890625, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 0.00357507965332711, + "pre_home_rank_cv_mean": 0.0021567674165372404, + "pre_home_rank_cv_min": 0.0006899225660317828, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 1.0064849853515625, + "pre_home_rank_imax_mean": 1.0033422311147053, + "pre_home_rank_imax_min": 1.0009918212890625, + "pre_home_rank_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "ultraep_drop": 96, + "ultraep_post": 1536, + "ultraep_pre": 1536 + }, + "required_capacity_max": 1.007126, + "required_capacity_mean": 1.0045282500000001, + "required_capacity_min": 1.002487, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "32", + "valid_max_rank_reduction_max": 0.0, + "valid_max_rank_reduction_mean": 0.0, + "valid_max_rank_reduction_min": 0.0, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + } +] \ No newline at end of file diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.tsv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..fa934b71e1e1a267204f829b3b2b98a809fb3b5e --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/probe_summary.tsv @@ -0,0 +1,12 @@ +run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches drop_expected_assignments drop_rate_pct_max drop_rate_pct_mean drop_rate_pct_min drop_rate_pct_n drop_rate_pct_overall dropped_tokens_sum_all_records duplicate_vector_mismatches global_snapshots incomplete_snapshots malformed_records observed_dropped_tokens peak_demand_max peak_demand_mean peak_demand_min peak_demand_n peak_rank_overflow_max peak_rank_overflow_mean peak_rank_overflow_min peak_rank_overflow_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop +G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G1 ultraep 8 2 1 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 25165824.0 2.891776 0.607152 0.0 96 0.6209174791971842 156259.0 0 192 0 0 156259.0 14128.0 10918.34375 8275.0 96 3888.0 1076.4583333333333 0.0 96 0.6189049804826644 0.08780981215252336 0.015802566559489504 192 1.724609375 1.063955307006836 1.010009765625 192 1.7320508075688772 1.4485647787782712 0.3594758110042677 192 4.0 3.6084569295247397 1.6087646484375 192 1.724609 1.33280565625 1.010132 96 0.7474516266861991 0.6845405986286021 0.27725927612110174 192 0 +G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 natural G1 ultraep 8 2 1 16 16 0 4096 16 static 1.8125 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 25165824.0 1.545951 0.03138885416666667 0.0 96 0.032329559326171875 8136.0 0 192 0 0 8136.0 18916.0 11069.770833333334 8275.0 96 4068.0 84.75 0.0 96 1.0236043430060333 0.09497773141059589 0.015224448662595366 192 2.30908203125 1.0747025807698567 1.010009765625 192 1.7320508075688772 1.444809403446171 0.3594758110042677 192 4.0 3.6151854197184243 1.6087646484375 192 2.309082 1.3512904166666668 1.010132 96 0.7474516266861991 0.6828229027074976 0.27725927612110174 192 0 +G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 natural G2 ultraep 32 8 4 16 16 0 4096 16 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 100663296.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 33099.0 33096.927083333336 33095.0 96 0.0 0.0 0.0 96 0.026494891363095952 0.013436746135527637 0.004813851711632 192 1.010101318359375 1.0100231170654297 1.010009765625 192 1.3185407730096428 0.7088423169085597 0.07964958686803787 192 3.99853515625 2.2032313346862793 1.17706298828125 192 1.010101 1.0100380104166666 1.009979 96 0.7474050555623397 0.5179521802733736 0.14192377495462796 192 1 +G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G2 ultraep 32 8 4 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 100663296.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 33099.0 33096.875 33096.0 96 0.0 0.0 0.0 96 0.026494891363095952 0.013604201092033112 0.004841919368473151 192 1.01007080078125 1.0100212097167969 1.010009765625 192 1.3181329746469168 0.7143701949255296 0.22890444224298007 192 3.99853515625 2.2135567665100098 1.33551025390625 192 1.010101 1.0100364479166666 1.01001 96 0.7474050555623397 0.5202877868268106 0.24370458388556282 192 1 +G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 forced G2 ultraep 32 8 4 16 16 0 4096 16 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 100663296.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 33096.0 33052.15625 32901.0 96 0.0 0.0 0.0 96 0.00717872106815807 0.004277032344446821 0.0016880800477701613 192 1.010009765625 1.006682554880778 1.002349853515625 192 0.007852555942790041 0.00436334506476056 0.0016880800477701613 192 1.01544189453125 1.0068944295247395 1.002349853515625 192 1.01001 1.0086718541666666 1.004059 96 0.005349522149425978 0.0002090865394183532 0.0 192 1 +G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 natural G3 ultraep 64 16 8 16 16 0 4096 16 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 201326592.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 66196.0 66193.28125 66192.0 96 0.0 0.0 0.0 96 0.026489120474252784 0.014233627693615905 0.004993448920418017 192 1.01007080078125 1.0100196202596028 1.010009765625 192 0.8287584673770169 0.47726214620952834 0.11019481148220499 192 2.6106719970703125 1.7842089335123699 1.1877593994140625 192 1.010071 1.0100292499999999 1.01001 96 0.6131226876610966 0.41320151666180793 0.14965121208617566 192 1 +G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G3 ultraep 64 16 8 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 201326592.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 66195.0 66193.34375 66192.0 96 0.0 0.0 0.0 96 0.026483352723689336 0.014464507564652654 0.00571681991985643 192 1.0100555419921875 1.0100204149882 1.010009765625 192 0.8316092132265209 0.4753321564816673 0.11019481148220499 192 2.8477020263671875 1.7751669883728027 1.1877593994140625 192 1.010056 1.01003021875 1.01001 96 0.6453085566397145 0.41069380212188733 0.14965121208617566 192 1 +G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 forced G3 ultraep 64 16 8 16 16 0 4096 16 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 201326592.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 66192.0 65961.36458333333 65746.0 96 0.0 0.0 0.0 96 0.005936610286960278 0.0030208295858681123 0.0014995787955522665 192 1.010009765625 1.004675070444743 1.001800537109375 192 0.005936610286960278 0.0030232844076121573 0.0014995787955522665 192 1.0113067626953125 1.0046818256378174 1.001800537109375 192 1.01001 1.0064905625 1.003204 96 0.0012824961902319055 6.679667657457841e-06 0.0 192 1 +G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1 natural G4 ultraep 128 32 16 16 16 0 4096 16 static 1.095703125 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 402653184.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 136700.0 132779.11458333334 132383.0 96 0.0 0.0 0.0 96 0.026463165595102558 0.014430625474517489 0.005656712434847103 192 1.0100250244140625 1.0100077788035076 1.0100021362304688 192 0.607811574382448 0.34190630383555115 0.08731759963810155 192 2.2518386840820312 1.561224063237508 1.121063232421875 192 1.042938 1.0130241979166665 1.010002 96 0.5514766917497027 0.3391114043117762 0.099060841159657 192 1 +G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1 natural G4 ultraep 128 32 16 16 16 0 4096 16 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 402653184.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 136700.0 132713.30208333334 132383.0 96 0.0 0.0 0.0 96 0.02646316532015595 0.01471717424328191 0.0057054839456758865 192 1.0100326538085938 1.0100078185399373 1.0100021362304688 192 0.6043235604226133 0.3401136548111448 0.08731759963810155 192 2.2515869140625 1.5504355827967327 1.121063232421875 192 1.042938 1.0125220937500001 1.010002 96 0.5514197614529683 0.3357737985870543 0.099060841159657 192 1 +G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 forced G4 ultraep 128 32 16 16 16 0 4096 16 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 402653184.0 0.0 0.0 0.0 96 0.0 0.0 0 192 0 0 0.0 132006.0 131665.52083333334 131398.0 96 0.0 0.0 0.0 96 0.00357507965332711 0.0021567674165372404 0.0006899225660317828 192 1.0064849853515625 1.0033422311147053 1.0009918212890625 192 0.00357507965332711 0.0021567674165372404 0.0006899225660317828 192 1.0064849853515625 1.0033422311147053 1.0009918212890625 192 1.007126 1.0045282500000001 1.002487 96 0.0 0.0 0.0 192 1 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/results.tsv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/results.tsv index 8456066104d1b0840c5e981771e4992428e14220..2123772ccc865e10458d2d10b3317b2f3f66c35a 100644 --- a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/results.tsv +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/results.tsv @@ -3,3 +3,10 @@ G1 ultraep matched 1 4096 16 1.25 none 1 OK 893.49 886.90 108731 /workspace/tier G1 ultraep matched 1 4096 16 1.8125 none 1 OK 914.84 903.60 113767 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 G2 ultraep matched 4 4096 16 1.25 none 1 OK 558.91 532.70 110343 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1 G2 ultraep matched 4 4096 16 1.0625 none 1 OK 551.94 527.00 107987 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 +G2 ultraep matched 4 4096 16 1.25 none+forcebal 1 OK 426.26 422.10 110343 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 +G3 ultraep matched 8 4096 16 1.25 none 1 OK 536.43 529.00 114319 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25/rep1 +G3 ultraep matched 8 4096 16 1.0625 none 1 OK 528.17 522.80 111567 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 +G3 ultraep matched 8 4096 16 1.25 none+forcebal 1 OK 472.34 470.60 114267 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1 +G4 ultraep matched 16 4096 16 1.25 none 1 OK 610.06 573.90 120487 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25/rep1 +G4 ultraep matched 16 4096 16 1.095703125 none 1 OK 594.23 563.90 118667 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/rep1 +G4 ultraep matched 16 4096 16 1.25 none+forcebal 1 OK 596.29 545.60 120487 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1